From 790374ede42634111d1c7a0fbf79f82195312145 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 13 Feb 2015 15:49:59 -0600 Subject: [PATCH 0001/2431] Add support for CASSANDRA-7660 This experimentally adds support for CASSANDRA-7660, which returns the indexes of bind markers that correspond to partition key columns in prepared statement response metadata. --- cassandra/cluster.py | 4 ++-- cassandra/protocol.py | 35 +++++++++++++++++++++++++++++++---- cassandra/query.py | 43 +++++++++++++++++++++++-------------------- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index b639af6992..40d414c743 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1534,13 +1534,13 @@ def prepare(self, query): future = ResponseFuture(self, message, query=None) try: future.send_request() - query_id, column_metadata = future.result(self.default_timeout) + query_id, column_metadata, pk_indexes = future.result(self.default_timeout) except Exception: log.exception("Error preparing query:") raise prepared_statement = PreparedStatement.from_message( - query_id, column_metadata, self.cluster.metadata, query, self.keyspace, + query_id, column_metadata, pk_indexes, self.cluster.metadata, query, self.keyspace, self._protocol_version) host = future._current_host diff --git a/cassandra/protocol.py b/cassandra/protocol.py index af279ef959..f570f487b2 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -559,7 +559,7 @@ def recv_body(cls, f, protocol_version, user_type_map): ksname = read_string(f) results = ksname elif kind == RESULT_KIND_PREPARED: - results = cls.recv_results_prepared(f, user_type_map) + results = cls.recv_results_prepared(f, protocol_version, user_type_map) elif kind == RESULT_KIND_SCHEMA_CHANGE: results = cls.recv_results_schema_change(f, protocol_version) return cls(kind, results, paging_state) @@ -578,16 +578,17 @@ def recv_results_rows(cls, f, protocol_version, user_type_map): return (paging_state, (colnames, parsed_rows)) @classmethod - def recv_results_prepared(cls, f, user_type_map): + def recv_results_prepared(cls, f, protocol_version, user_type_map): query_id = read_binary_string(f) - _, column_metadata = cls.recv_results_metadata(f, user_type_map) - return (query_id, column_metadata) + column_metadata, pk_indexes = cls.recv_prepared_metadata(f, protocol_version, user_type_map) + return (query_id, column_metadata, pk_indexes) @classmethod def recv_results_metadata(cls, f, user_type_map): flags = read_int(f) glob_tblspec = bool(flags & cls._FLAGS_GLOBAL_TABLES_SPEC) colcount = read_int(f) + if flags & cls._HAS_MORE_PAGES_FLAG: paging_state = read_binary_longstring(f) else: @@ -608,6 +609,32 @@ def recv_results_metadata(cls, f, user_type_map): column_metadata.append((colksname, colcfname, colname, coltype)) return paging_state, column_metadata + @classmethod + def recv_prepared_metadata(cls, f, protocol_version, user_type_map): + flags = read_int(f) + glob_tblspec = bool(flags & cls._FLAGS_GLOBAL_TABLES_SPEC) + colcount = read_int(f) + pk_indexes = None + if protocol_version >= 4: + num_pk_indexes = read_int(f) + pk_indexes = [read_short(f) for _ in range(num_pk_indexes)] + + if glob_tblspec: + ksname = read_string(f) + cfname = read_string(f) + column_metadata = [] + for _ in range(colcount): + if glob_tblspec: + colksname = ksname + colcfname = cfname + else: + colksname = read_string(f) + colcfname = read_string(f) + colname = read_string(f) + coltype = cls.read_type(f, user_type_map) + column_metadata.append((colksname, colcfname, colname, coltype)) + return column_metadata, pk_indexes + @classmethod def recv_results_schema_change(cls, f, protocol_version): return EventMessage.recv_schema_change(f, protocol_version) diff --git a/cassandra/query.py b/cassandra/query.py index c159e7877b..0f7db27ed8 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -353,29 +353,32 @@ def __init__(self, column_metadata, query_id, routing_key_indexes, query, keyspa self.fetch_size = fetch_size @classmethod - def from_message(cls, query_id, column_metadata, cluster_metadata, query, prepared_keyspace, protocol_version): + def from_message(cls, query_id, column_metadata, pk_indexes, cluster_metadata, query, prepared_keyspace, protocol_version): if not column_metadata: return PreparedStatement(column_metadata, query_id, None, query, prepared_keyspace, protocol_version) - partition_key_columns = None - routing_key_indexes = None - - ks_name, table_name, _, _ = column_metadata[0] - ks_meta = cluster_metadata.keyspaces.get(ks_name) - if ks_meta: - table_meta = ks_meta.tables.get(table_name) - if table_meta: - partition_key_columns = table_meta.partition_key - - # make a map of {column_name: index} for each column in the statement - statement_indexes = dict((c[2], i) for i, c in enumerate(column_metadata)) - - # a list of which indexes in the statement correspond to partition key items - try: - routing_key_indexes = [statement_indexes[c.name] - for c in partition_key_columns] - except KeyError: # we're missing a partition key component in the prepared - pass # statement; just leave routing_key_indexes as None + if pk_indexes: + routing_key_indexes = pk_indexes + else: + partition_key_columns = None + routing_key_indexes = None + + ks_name, table_name, _, _ = column_metadata[0] + ks_meta = cluster_metadata.keyspaces.get(ks_name) + if ks_meta: + table_meta = ks_meta.tables.get(table_name) + if table_meta: + partition_key_columns = table_meta.partition_key + + # make a map of {column_name: index} for each column in the statement + statement_indexes = dict((c[2], i) for i, c in enumerate(column_metadata)) + + # a list of which indexes in the statement correspond to partition key items + try: + routing_key_indexes = [statement_indexes[c.name] + for c in partition_key_columns] + except KeyError: # we're missing a partition key component in the prepared + pass # statement; just leave routing_key_indexes as None return PreparedStatement(column_metadata, query_id, routing_key_indexes, query, prepared_keyspace, protocol_version) From 2d5466f8a2cb384ec96abc1720c8418faa542cd3 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Mon, 9 Mar 2015 16:58:58 -0500 Subject: [PATCH 0002/2431] Add Read and WriteFailure error classes --- cassandra/protocol.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index ca628a433d..8b22f75d97 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -263,6 +263,35 @@ def to_exception(self): return ReadTimeout(self.summary_msg(), **self.info) +class ReadFailureMessage(RequestExecutionException): + summary = "Replica(s) failed to execute read" + error_code = 0x1300 + + @staticmethod + def recv_error_info(f): + return { + 'consistency': read_consistency_level(f), + 'received_responses': read_int(f), + 'required_responses': read_int(f), + 'failures': read_int(f), + 'data_retrieved': bool(read_byte(f)), + } + + +class WriteFailureMessage(RequestExecutionException): + summary = "Replica(s) failed to execute write" + error_code = 0x1500 + + @staticmethod + def recv_error_info(f): + return { + 'consistency': read_consistency_level(f), + 'received_responses': read_int(f), + 'required_responses': read_int(f), + 'failures': read_int(f), + 'write_type': WriteType.name_to_value[read_string(f)], + } + class SyntaxException(RequestValidationException): summary = 'Syntax error in CQL query' error_code = 0x2000 From 56b8a1463c61e8f0859941b7ae62a37a55470a2d Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Tue, 10 Mar 2015 11:56:13 +0800 Subject: [PATCH 0003/2431] Added WriteFailure and ReadFailure exceptions --- cassandra/__init__.py | 70 +++++++++++++++++++++++++++++++++++++++++++ cassandra/protocol.py | 7 +++++ 2 files changed, 77 insertions(+) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 96f085767c..909eeb13da 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -221,6 +221,76 @@ def __init__(self, message, write_type=None, **kwargs): self.write_type = write_type +class Failure(Exception): + """ + Replicas sent a failure to the coordinator. + """ + + consistency = None + """ The requested :class:`ConsistencyLevel` """ + + required_responses = None + """ The number of required replica responses """ + + received_responses = None + """ + The number of replicas that responded before the coordinator timed out + the operation + """ + + failures = None + """ + The number of replicas that sent a failure message + """ + + def __init__(self, summary_message, consistency=None, required_responses=None, received_responses=None, failures=None): + self.consistency = consistency + self.required_responses = required_responses + self.received_responses = received_responses + self.failures = failures + Exception.__init__(self, summary_message + ' info=' + + repr({'consistency': consistency_value_to_name(consistency), + 'required_responses': required_responses, + 'received_responses': received_responses, + 'failures' : failures})) + + +class ReadFailure(Failure): + """ + A subclass of :exc:`Failure` for read operations. + + This indicates that the replicas sent a failure message to the coordinator. + """ + + data_retrieved = None + """ + A boolean indicating whether the requested data was retrieved + by the coordinator from any replicas before it timed out the + operation + """ + + def __init__(self, message, data_retrieved=None, **kwargs): + Failure.__init__(self, message, **kwargs) + self.data_retrieved = data_retrieved + + +class WriteFailure(Failure): + """ + A subclass of :exc:`Failure` for write operations. + + This indicates that the replicas sent a failure message to the coordinator. + """ + + write_type = None + """ + The type of write operation, enum on :class:`~cassandra.policies.WriteType` + """ + + def __init__(self, message, write_type=None, **kwargs): + Failure.__init__(self, message, **kwargs) + self.write_type = write_type + + class AlreadyExists(Exception): """ An attempt was made to create a keyspace or table that already exists. diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 8b22f75d97..7f5c2cea2c 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -22,6 +22,7 @@ import io from cassandra import (Unavailable, WriteTimeout, ReadTimeout, + WriteFailure, ReadFailure, AlreadyExists, InvalidRequest, Unauthorized, UnsupportedOperation) from cassandra.marshal import (int32_pack, int32_unpack, uint16_pack, uint16_unpack, @@ -277,6 +278,9 @@ def recv_error_info(f): 'data_retrieved': bool(read_byte(f)), } + def to_exception(self): + return ReadFailure(self.summary_msg(), **self.info) + class WriteFailureMessage(RequestExecutionException): summary = "Replica(s) failed to execute write" @@ -292,6 +296,9 @@ def recv_error_info(f): 'write_type': WriteType.name_to_value[read_string(f)], } + def to_exception(self): + return WriteFailure(self.summary_msg(), **self.info) + class SyntaxException(RequestValidationException): summary = 'Syntax error in CQL query' error_code = 0x2000 From 7a6e72d79177ed3ae8a551bca488dacdf098abb9 Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Fri, 20 Mar 2015 11:35:13 +0800 Subject: [PATCH 0004/2431] CASSANDRA-7814: Added indexes to table and ks metadata and export_as_string() to IndexMetadata --- cassandra/metadata.py | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 1f88d8f9e6..9b01a18c8e 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -122,10 +122,9 @@ def rebuild_schema(self, ks_results, type_results, cf_results, col_results, trig keyspace_col_rows = col_def_rows.get(keyspace_meta.name, {}) keyspace_trigger_rows = trigger_rows.get(keyspace_meta.name, {}) for table_row in cf_def_rows.get(keyspace_meta.name, []): - table_meta = self._build_table_metadata( + self._add_table_metadata_to_ks( keyspace_meta, table_row, keyspace_col_rows, keyspace_trigger_rows) - keyspace_meta.tables[table_meta.name] = table_meta for usertype_row in usertype_rows.get(keyspace_meta.name, []): usertype = self._build_usertype(keyspace_meta.name, usertype_row) @@ -184,10 +183,12 @@ def table_changed(self, keyspace, table, cf_results, col_results, triggers_resul if not cf_results: # the table was removed - keyspace_meta.tables.pop(table, None) + table_meta = keyspace_meta.tables.pop(table, None) + if table_meta: + self._clear_table_indexes_in_ks(keyspace_meta, table_meta.name) else: assert len(cf_results) == 1 - keyspace_meta.tables[table] = self._build_table_metadata( + self._add_table_metadata_to_ks( keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) @@ -215,6 +216,20 @@ def _build_usertype(self, keyspace, usertype_row): return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], usertype_row['field_names'], type_classes) + def _add_table_metadata_to_ks(self, keyspace_metadata, row, col_rows, trigger_rows): + self._clear_table_indexes_in_ks(keyspace_metadata, row["columnfamily_name"]) + + table_metadata = self._build_table_metadata(keyspace_metadata, row, col_rows, trigger_rows) + keyspace_metadata.tables[table_metadata.name] = table_metadata + for index_name, index_metadata in table_metadata.indexes.iteritems(): + keyspace_metadata.indexes[index_name] = index_metadata + + def _clear_table_indexes_in_ks(self, keyspace_metadata, table_name): + if table_name in keyspace_metadata.tables: + table_meta = keyspace_metadata.tables[table_name] + for index_name in table_meta.indexes: + keyspace_metadata.indexes.pop(index_name, None) + def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) @@ -385,6 +400,8 @@ def _build_column_metadata(self, table_metadata, row): column_meta = ColumnMetadata(table_metadata, name, data_type, is_static=is_static) index_meta = self._build_index_metadata(column_meta, row) column_meta.index = index_meta + if index_meta: + table_metadata.indexes[index_meta.name] = index_meta return column_meta def _build_index_metadata(self, column_metadata, row): @@ -733,6 +750,11 @@ class KeyspaceMetadata(object): A map from table names to instances of :class:`~.TableMetadata`. """ + indexes = None + """ + A dict mapping index names to :class:`.IndexMetadata` instances. + """ + user_types = None """ A map from user-defined type names to instances of :class:`~cassandra.metadata..UserType`. @@ -745,6 +767,7 @@ def __init__(self, name, durable_writes, strategy_class, strategy_options): self.durable_writes = durable_writes self.replication_strategy = ReplicationStrategy.create(strategy_class, strategy_options) self.tables = {} + self.indexes = {} self.user_types = {} def export_as_string(self): @@ -884,6 +907,11 @@ def primary_key(self): A dict mapping column names to :class:`.ColumnMetadata` instances. """ + indexes = None + """ + A dict mapping index names to :class:`.IndexMetadata` instances. + """ + is_compact_storage = False options = None @@ -945,6 +973,7 @@ def __init__(self, keyspace_metadata, name, partition_key=None, clustering_key=N self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key self.columns = OrderedDict() if columns is None else columns + self.indexes = {} self.options = options self.comparator = None self.triggers = OrderedDict() if triggers is None else triggers @@ -1242,6 +1271,12 @@ def as_cql_query(self): protect_name(self.column.name), self.index_options["class_name"]) + def export_as_string(self): + """ + Returns a CQL query string that can be used to recreate this index. + """ + return self.as_cql_query() + ';' + class TokenMap(object): """ From 521ab1a116f25bbb34a33b278c897eec03018454 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 2 Apr 2015 20:49:48 -0700 Subject: [PATCH 0005/2431] refactored MetricsTests --- tests/integration/standard/test_metrics.py | 125 +++++++++++---------- 1 file changed, 64 insertions(+), 61 deletions(-) diff --git a/tests/integration/standard/test_metrics.py b/tests/integration/standard/test_metrics.py index 674727eae6..1996495c0b 100644 --- a/tests/integration/standard/test_metrics.py +++ b/tests/integration/standard/test_metrics.py @@ -35,27 +35,28 @@ class MetricsTests(unittest.TestCase): def test_connection_error(self): """ Trigger and ensure connection_errors are counted + Stop all node with the driver knowing about the "DOWN" states. """ - cluster = Cluster(metrics_enabled=True, - protocol_version=PROTOCOL_VERSION) - session = cluster.connect() - session.execute("USE test3rf") + cluster = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION) + session = cluster.connect("test3rf") # Test writes for i in range(0, 100): - session.execute_async( - """ - INSERT INTO test3rf.test (k, v) VALUES (%s, %s) - """ % (i, i)) + session.execute_async("INSERT INTO test (k, v) VALUES ({0}, {1})".format(i, i)) - # Force kill cluster + # Stop the cluster get_cluster().stop(wait=True, gently=False) + try: # Ensure the nodes are actually down - self.assertRaises(NoHostAvailable, session.execute, "USE test3rf") + query = SimpleStatement("SELECT * FROM test", consistency_level=ConsistencyLevel.ALL) + with self.assertRaises(NoHostAvailable): + session.execute(query) finally: get_cluster().start(wait_for_binary_proto=True, wait_other_notice=True) + # Give some time for the cluster to come back up, for the next test + time.sleep(5) self.assertGreater(cluster.metrics.stats.connection_errors, 0) cluster.shutdown() @@ -63,115 +64,117 @@ def test_connection_error(self): def test_write_timeout(self): """ Trigger and ensure write_timeouts are counted - Write a key, value pair. Force kill a node without waiting for the cluster to register the death. + Write a key, value pair. Pause a node without the coordinator node knowing about the "DOWN" state. Attempt a write at cl.ALL and receive a WriteTimeout. """ - cluster = Cluster(metrics_enabled=True, - protocol_version=PROTOCOL_VERSION) - session = cluster.connect() + cluster = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION) + session = cluster.connect("test3rf") # Test write - session.execute("INSERT INTO test3rf.test (k, v) VALUES (1, 1)") + session.execute("INSERT INTO test (k, v) VALUES (1, 1)") # Assert read - query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL) - results = session.execute(query, {'k': 1}) - self.assertEqual(1, results[0].v) + query = SimpleStatement("SELECT * FROM test WHERE k=1", consistency_level=ConsistencyLevel.ALL) + results = session.execute(query) + self.assertEqual(1, len(results)) - # Force kill ccm node - get_node(1).stop(wait=False, gently=False) + # Pause node so it shows as unreachable to coordinator + get_node(1).pause() try: # Test write - query = SimpleStatement("INSERT INTO test3rf.test (k, v) VALUES (2, 2)", consistency_level=ConsistencyLevel.ALL) - self.assertRaises(WriteTimeout, session.execute, query, timeout=None) + query = SimpleStatement("INSERT INTO test (k, v) VALUES (2, 2)", consistency_level=ConsistencyLevel.ALL) + with self.assertRaises(WriteTimeout): + session.execute(query) self.assertEqual(1, cluster.metrics.stats.write_timeouts) finally: - get_node(1).start(wait_other_notice=True, wait_for_binary_proto=True) + get_node(1).resume() cluster.shutdown() def test_read_timeout(self): """ Trigger and ensure read_timeouts are counted - Write a key, value pair. Force kill a node without waiting for the cluster to register the death. + Write a key, value pair. Pause a node without the coordinator node knowing about the "DOWN" state. Attempt a read at cl.ALL and receive a ReadTimeout. """ - cluster = Cluster(metrics_enabled=True, - protocol_version=PROTOCOL_VERSION) - session = cluster.connect() + cluster = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION) + session = cluster.connect("test3rf") # Test write - session.execute("INSERT INTO test3rf.test (k, v) VALUES (1, 1)") + session.execute("INSERT INTO test (k, v) VALUES (1, 1)") # Assert read - query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL) - results = session.execute(query, {'k': 1}) - self.assertEqual(1, results[0].v) + query = SimpleStatement("SELECT * FROM test WHERE k=1", consistency_level=ConsistencyLevel.ALL) + results = session.execute(query) + self.assertEqual(1, len(results)) - # Force kill ccm node - get_node(1).stop(wait=False, gently=False) + # Pause node so it shows as unreachable to coordinator + get_node(1).pause() try: # Test read - query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL) - self.assertRaises(ReadTimeout, session.execute, query, {'k': 1}, timeout=None) + query = SimpleStatement("SELECT * FROM test", consistency_level=ConsistencyLevel.ALL) + with self.assertRaises(ReadTimeout): + session.execute(query, timeout=None) self.assertEqual(1, cluster.metrics.stats.read_timeouts) finally: - get_node(1).start(wait_other_notice=True, wait_for_binary_proto=True) + get_node(1).resume() cluster.shutdown() def test_unavailable(self): """ Trigger and ensure unavailables are counted - Write a key, value pair. Kill a node while waiting for the cluster to register the death. + Write a key, value pair. Stop a node with the coordinator node knowing about the "DOWN" state. Attempt an insert/read at cl.ALL and receive a Unavailable Exception. """ - cluster = Cluster(metrics_enabled=True, - protocol_version=PROTOCOL_VERSION) - session = cluster.connect() + cluster = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION) + session = cluster.connect("test3rf") # Test write - session.execute("INSERT INTO test3rf.test (k, v) VALUES (1, 1)") + session.execute("INSERT INTO test (k, v) VALUES (1, 1)") # Assert read - query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL) - results = session.execute(query, {'k': 1}) - self.assertEqual(1, results[0].v) + query = SimpleStatement("SELECT * FROM test WHERE k=1", consistency_level=ConsistencyLevel.ALL) + results = session.execute(query) + self.assertEqual(1, len(results)) - # Force kill ccm node + # Stop node gracefully get_node(1).stop(wait=True, gently=True) - time.sleep(5) try: # Test write - query = SimpleStatement("INSERT INTO test3rf.test (k, v) VALUES (2, 2)", consistency_level=ConsistencyLevel.ALL) - self.assertRaises(Unavailable, session.execute, query) + query = SimpleStatement("INSERT INTO test (k, v) VALUES (2, 2)", consistency_level=ConsistencyLevel.ALL) + with self.assertRaises(Unavailable): + session.execute(query) self.assertEqual(1, cluster.metrics.stats.unavailables) # Test write - query = SimpleStatement("SELECT v FROM test3rf.test WHERE k=%(k)s", consistency_level=ConsistencyLevel.ALL) - self.assertRaises(Unavailable, session.execute, query, {'k': 1}) + query = SimpleStatement("SELECT * FROM test", consistency_level=ConsistencyLevel.ALL) + with self.assertRaises(Unavailable): + session.execute(query, timeout=None) self.assertEqual(2, cluster.metrics.stats.unavailables) finally: get_node(1).start(wait_other_notice=True, wait_for_binary_proto=True) + # Give some time for the cluster to come back up, for the next test + time.sleep(5) cluster.shutdown() - def test_other_error(self): - # TODO: Bootstrapping or Overloaded cases - pass - - def test_ignore(self): - # TODO: Look for ways to generate ignores - pass - - def test_retry(self): - # TODO: Look for ways to generate retries - pass + # def test_other_error(self): + # # TODO: Bootstrapping or Overloaded cases + # pass + # + # def test_ignore(self): + # # TODO: Look for ways to generate ignores + # pass + # + # def test_retry(self): + # # TODO: Look for ways to generate retries + # pass From 73ec60f606d793200b360bcac314aecaec3bf9b9 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Tue, 7 Apr 2015 13:27:16 -0700 Subject: [PATCH 0006/2431] Refactored TypeTests --- tests/integration/datatype_utils.py | 75 +-- tests/integration/standard/test_types.py | 685 +++++++++++------------ 2 files changed, 369 insertions(+), 391 deletions(-) diff --git a/tests/integration/datatype_utils.py b/tests/integration/datatype_utils.py index 41a4c09e01..bd76c36fec 100644 --- a/tests/integration/datatype_utils.py +++ b/tests/integration/datatype_utils.py @@ -11,22 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from decimal import Decimal -import datetime -from uuid import UUID -import pytz +from datetime import datetime, date, time +from uuid import uuid1, uuid4 try: from blist import sortedset except ImportError: sortedset = set # noqa -DATA_TYPE_PRIMITIVES = [ +from cassandra.util import OrderedMap + +from tests.integration import get_server_versions + + +PRIMITIVE_DATATYPES = [ 'ascii', 'bigint', 'blob', 'boolean', - # 'counter', counters are not allowed inside tuples 'decimal', 'double', 'float', @@ -40,22 +44,28 @@ 'varint', ] -DATA_TYPE_NON_PRIMITIVE_NAMES = [ +COLLECTION_TYPES = [ 'list', 'set', 'map', - 'tuple' ] -def get_sample_data(): - """ - Create a standard set of sample inputs for testing. - """ +def update_datatypes(): + _cass_version, _cql_version = get_server_versions() + + if _cass_version >= (2, 1, 0): + COLLECTION_TYPES.append('tuple') + + if _cass_version >= (2, 1, 5): + PRIMITIVE_DATATYPES.append('date') + PRIMITIVE_DATATYPES.append('time') + +def get_sample_data(): sample_data = {} - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: if datatype == 'ascii': sample_data[datatype] = 'ascii' @@ -68,10 +78,6 @@ def get_sample_data(): elif datatype == 'boolean': sample_data[datatype] = True - elif datatype == 'counter': - # Not supported in an insert statement - pass - elif datatype == 'decimal': sample_data[datatype] = Decimal('12.3E+7') @@ -91,13 +97,13 @@ def get_sample_data(): sample_data[datatype] = 'text' elif datatype == 'timestamp': - sample_data[datatype] = datetime.datetime.fromtimestamp(872835240, tz=pytz.timezone('America/New_York')).astimezone(pytz.UTC).replace(tzinfo=None) + sample_data[datatype] = datetime(2013, 12, 31, 23, 59, 59, 999000) elif datatype == 'timeuuid': - sample_data[datatype] = UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66') + sample_data[datatype] = uuid1() elif datatype == 'uuid': - sample_data[datatype] = UUID('067e6162-3b6f-4ae2-a171-2470b63dff00') + sample_data[datatype] = uuid4() elif datatype == 'varchar': sample_data[datatype] = 'varchar' @@ -105,35 +111,40 @@ def get_sample_data(): elif datatype == 'varint': sample_data[datatype] = int(str(2147483647) + '000') + elif datatype == 'date': + sample_data[datatype] = date(2015, 1, 15) + + elif datatype == 'time': + sample_data[datatype] = time(16, 47, 25, 7) + else: - raise Exception('Missing handling of %s.' % datatype) + raise Exception("Missing handling of {0}".format(datatype)) return sample_data SAMPLE_DATA = get_sample_data() + def get_sample(datatype): """ - Helper method to access created sample data + Helper method to access created sample data for primitive types """ return SAMPLE_DATA[datatype] -def get_nonprim_sample(non_prim_type, datatype): + +def get_collection_sample(collection_type, datatype): """ - Helper method to access created sample data for non-primitives + Helper method to access created sample data for collection types """ - if non_prim_type == 'list': + if collection_type == 'list': return [get_sample(datatype), get_sample(datatype)] - elif non_prim_type == 'set': + elif collection_type == 'set': return sortedset([get_sample(datatype)]) - elif non_prim_type == 'map': - if datatype == 'blob': - return {get_sample('ascii'): get_sample(datatype)} - else: - return {get_sample(datatype): get_sample(datatype)} - elif non_prim_type == 'tuple': + elif collection_type == 'map': + return OrderedMap([(get_sample(datatype), get_sample(datatype))]) + elif collection_type == 'tuple': return (get_sample(datatype),) else: - raise Exception('Missing handling of non-primitive type {0}.'.format(non_prim_type)) + raise Exception('Missing handling of non-primitive type {0}.'.format(collection_type)) diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index 2191db1f41..b256e87595 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from tests.integration.datatype_utils import get_sample, DATA_TYPE_PRIMITIVES, DATA_TYPE_NON_PRIMITIVE_NAMES try: import unittest2 as unittest @@ -21,85 +20,53 @@ import logging log = logging.getLogger(__name__) -from collections import namedtuple -from decimal import Decimal -from datetime import datetime, date, time -from functools import partial +from datetime import datetime import six -from uuid import uuid1, uuid4 from cassandra import InvalidRequest from cassandra.cluster import Cluster from cassandra.cqltypes import Int32Type, EMPTY -from cassandra.query import dict_factory -from cassandra.util import OrderedMap, sortedset +from cassandra.query import dict_factory, ordered_dict_factory +from cassandra.util import sortedset from tests.integration import get_server_versions, use_singledc, PROTOCOL_VERSION - -# defined in module scope for pickling in OrderedMap -nested_collection_udt = namedtuple('nested_collection_udt', ['m', 't', 'l', 's']) -nested_collection_udt_nested = namedtuple('nested_collection_udt_nested', ['m', 't', 'l', 's', 'u']) +from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, COLLECTION_TYPES, \ + get_sample, get_collection_sample def setup_module(): use_singledc() + update_datatypes() class TypeTests(unittest.TestCase): - _types_table_created = False - - @classmethod - def setup_class(cls): - cls._cass_version, cls._cql_version = get_server_versions() - - cls._col_types = ['text', - 'ascii', - 'bigint', - 'boolean', - 'decimal', - 'double', - 'float', - 'inet', - 'int', - 'list', - 'set', - 'map', - 'timestamp', - 'uuid', - 'timeuuid', - 'varchar', - 'varint'] - - if cls._cass_version >= (2, 1, 4): - cls._col_types.extend(('date', 'time')) - - cls._session = Cluster(protocol_version=PROTOCOL_VERSION).connect() - cls._session.execute("CREATE KEYSPACE typetests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") - cls._session.set_keyspace("typetests") - - @classmethod - def teardown_class(cls): - cls._session.execute("DROP KEYSPACE typetests") - cls._session.cluster.shutdown() - - def test_blob_type_as_string(self): - s = self._session + def setUp(self): + self._cass_version, self._cql_version = get_server_versions() - s.execute(""" - CREATE TABLE blobstring ( - a ascii, - b blob, - PRIMARY KEY (a) - ) - """) + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + self.session.execute("CREATE KEYSPACE typetests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") + self.cluster.shutdown() + + def tearDown(self): + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + self.session.execute("DROP KEYSPACE typetests") + self.cluster.shutdown() + + def test_can_insert_blob_type_as_string(self): + """ + Tests that blob type in Cassandra does not map to string in Python + """ - params = [ - 'key1', - b'blobyblob' - ] + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") - query = 'INSERT INTO blobstring (a, b) VALUES (%s, %s)' + s.execute("CREATE TABLE blobstring (a ascii PRIMARY KEY, b blob)") + + params = ['key1', b'blobyblob'] + query = "INSERT INTO blobstring (a, b) VALUES (%s, %s)" # In python 3, the 'bytes' type is treated as a blob, so we can # correctly encode it with hex notation. @@ -118,243 +85,277 @@ def test_blob_type_as_string(self): params[1] = params[1].encode('hex') s.execute(query, params) - expected_vals = [ - 'key1', - bytearray(b'blobyblob') - ] - - results = s.execute("SELECT * FROM blobstring") - for expected, actual in zip(expected_vals, results[0]): + results = s.execute("SELECT * FROM blobstring")[0] + for expected, actual in zip(params, results): self.assertEqual(expected, actual) - def test_blob_type_as_bytearray(self): - s = self._session - s.execute(""" - CREATE TABLE blobbytes ( - a ascii, - b blob, - PRIMARY KEY (a) - ) - """) - - params = [ - 'key1', - bytearray(b'blob1') - ] - - query = 'INSERT INTO blobbytes (a, b) VALUES (%s, %s);' - s.execute(query, params) + c.shutdown() - expected_vals = [ - 'key1', - bytearray(b'blob1') - ] + def test_can_insert_blob_type_as_bytearray(self): + """ + Tests that blob type in Cassandra maps to bytearray in Python + """ + + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") + + s.execute("CREATE TABLE blobbytes (a ascii PRIMARY KEY, b blob)") - results = s.execute("SELECT * FROM blobbytes") + params = ['key1', bytearray(b'blob1')] + s.execute("INSERT INTO blobbytes (a, b) VALUES (%s, %s)", params) - for expected, actual in zip(expected_vals, results[0]): + results = s.execute("SELECT * FROM blobbytes")[0] + for expected, actual in zip(params, results): self.assertEqual(expected, actual) - def _create_all_types_table(self): - if not self._types_table_created: - TypeTests._col_names = ["%s_col" % col_type.translate(None, '<> ,') for col_type in self._col_types] - cql = "CREATE TABLE alltypes ( key int PRIMARY KEY, %s)" % ','.join("%s %s" % name_type for name_type in zip(self._col_names, self._col_types)) - self._session.execute(cql) - TypeTests._types_table_created = True - - def test_basic_types(self): - - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) - - self._create_all_types_table() - - v1_uuid = uuid1() - v4_uuid = uuid4() - mydatetime = datetime(2013, 12, 31, 23, 59, 59, 999000) - - # this could use some rework tying column types to names (instead of relying on position) - params = [ - "text", - "ascii", - 12345678923456789, # bigint - True, # boolean - Decimal('1.234567890123456789'), # decimal - 0.000244140625, # double - 1.25, # float - "1.2.3.4", # inet - 12345, # int - ['a', 'b', 'c'], # list collection - set([1, 2, 3]), # set collection - {'a': 1, 'b': 2}, # map collection - mydatetime, # timestamp - v4_uuid, # uuid - v1_uuid, # timeuuid - u"sometext\u1234", # varchar - 123456789123456789123456789, # varint - ] - - expected_vals = [ - "text", - "ascii", - 12345678923456789, # bigint - True, # boolean - Decimal('1.234567890123456789'), # decimal - 0.000244140625, # double - 1.25, # float - "1.2.3.4", # inet - 12345, # int - ['a', 'b', 'c'], # list collection - sortedset((1, 2, 3)), # set collection - {'a': 1, 'b': 2}, # map collection - mydatetime, # timestamp - v4_uuid, # uuid - v1_uuid, # timeuuid - u"sometext\u1234", # varchar - 123456789123456789123456789, # varint - ] - - if self._cass_version >= (2, 1, 4): - mydate = date(2015, 1, 15) - mytime = time(16, 47, 25, 7) - - params.append(mydate) - params.append(mytime) - - expected_vals.append(mydate) - expected_vals.append(mytime) - - columns_string = ','.join(self._col_names) - placeholders = ', '.join(["%s"] * len(self._col_names)) - s.execute("INSERT INTO alltypes (key, %s) VALUES (0, %s)" % - (columns_string, placeholders), params) - - results = s.execute("SELECT %s FROM alltypes WHERE key=0" % columns_string) - - for expected, actual in zip(expected_vals, results[0]): + c.shutdown() + + def test_can_insert_primitive_datatypes(self): + """ + Test insertion of all datatype primitives + """ + + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") + + # create table + alpha_type_list = ["zz int PRIMARY KEY"] + col_names = ["zz"] + start_index = ord('a') + for i, datatype in enumerate(PRIMITIVE_DATATYPES): + alpha_type_list.append("{0} {1}".format(chr(start_index + i), datatype)) + col_names.append(chr(start_index + i)) + + s.execute("CREATE TABLE alltypes ({0})".format(', '.join(alpha_type_list))) + + # create the input + params = [0] + for datatype in PRIMITIVE_DATATYPES: + params.append((get_sample(datatype))) + + # insert into table as a simple statement + columns_string = ', '.join(col_names) + placeholders = ', '.join(["%s"] * len(col_names)) + s.execute("INSERT INTO alltypes ({0}) VALUES ({1})".format(columns_string, placeholders), params) + + # verify data + results = s.execute("SELECT {0} FROM alltypes WHERE zz=0".format(columns_string))[0] + for expected, actual in zip(params, results): self.assertEqual(actual, expected) # try the same thing with a prepared statement - placeholders = ','.join(["?"] * len(self._col_names)) - prepared = s.prepare("INSERT INTO alltypes (key, %s) VALUES (1, %s)" % - (columns_string, placeholders)) - s.execute(prepared.bind(params)) + placeholders = ','.join(["?"] * len(col_names)) + insert = s.prepare("INSERT INTO alltypes ({0}) VALUES ({1})".format(columns_string, placeholders)) + s.execute(insert.bind(params)) + + # verify data + results = s.execute("SELECT {0} FROM alltypes WHERE zz=0".format(columns_string))[0] + for expected, actual in zip(params, results): + self.assertEqual(actual, expected) + + # verify data with prepared statement query + select = s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)) + results = s.execute(select.bind([0]))[0] + for expected, actual in zip(params, results): + self.assertEqual(actual, expected) - results = s.execute("SELECT %s FROM alltypes WHERE key=1" % columns_string) + # verify data with with prepared statement, use dictionary with no explicit columns + s.row_factory = ordered_dict_factory + select = s.prepare("SELECT * FROM alltypes") + results = s.execute(select)[0] - for expected, actual in zip(expected_vals, results[0]): + for expected, actual in zip(params, results.values()): self.assertEqual(actual, expected) - # query with prepared statement - prepared = s.prepare("SELECT %s FROM alltypes WHERE key=?" % columns_string) - results = s.execute(prepared.bind((1,))) + c.shutdown() + + def test_can_insert_collection_datatypes(self): + """ + Test insertion of all collection types + """ - for expected, actual in zip(expected_vals, results[0]): + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") + # use tuple encoding, to convert native python tuple into raw CQL + s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple + + # create table + alpha_type_list = ["zz int PRIMARY KEY"] + col_names = ["zz"] + start_index = ord('a') + for i, collection_type in enumerate(COLLECTION_TYPES): + for j, datatype in enumerate(PRIMITIVE_DATATYPES): + if collection_type == "map": + type_string = "{0}_{1} {2}<{3}, {3}>".format(chr(start_index + i), chr(start_index + j), + collection_type, datatype) + elif collection_type == "tuple": + type_string = "{0}_{1} frozen<{2}<{3}>>".format(chr(start_index + i), chr(start_index + j), + collection_type, datatype) + else: + type_string = "{0}_{1} {2}<{3}>".format(chr(start_index + i), chr(start_index + j), + collection_type, datatype) + alpha_type_list.append(type_string) + col_names.append("{0}_{1}".format(chr(start_index + i), chr(start_index + j))) + + s.execute("CREATE TABLE allcoltypes ({0})".format(', '.join(alpha_type_list))) + columns_string = ', '.join(col_names) + + # create the input for simple statement + params = [0] + for collection_type in COLLECTION_TYPES: + for datatype in PRIMITIVE_DATATYPES: + params.append((get_collection_sample(collection_type, datatype))) + + # insert into table as a simple statement + placeholders = ', '.join(["%s"] * len(col_names)) + s.execute("INSERT INTO allcoltypes ({0}) VALUES ({1})".format(columns_string, placeholders), params) + + # verify data + results = s.execute("SELECT {0} FROM allcoltypes WHERE zz=0".format(columns_string))[0] + for expected, actual in zip(params, results): self.assertEqual(actual, expected) - # query with prepared statement, no explicit columns - s.row_factory = dict_factory - prepared = s.prepare("SELECT * FROM alltypes") - results = s.execute(prepared.bind(())) + # create the input for prepared statement + params = [0] + for collection_type in COLLECTION_TYPES: + for datatype in PRIMITIVE_DATATYPES: + params.append((get_collection_sample(collection_type, datatype))) - row = results[0] - for expected, name in zip(expected_vals, self._col_names): - self.assertEqual(row[name], expected) + # try the same thing with a prepared statement + placeholders = ','.join(["?"] * len(col_names)) + insert = s.prepare("INSERT INTO allcoltypes ({0}) VALUES ({1})".format(columns_string, placeholders)) + s.execute(insert.bind(params)) - s.shutdown() + # verify data + results = s.execute("SELECT {0} FROM allcoltypes WHERE zz=0".format(columns_string))[0] + for expected, actual in zip(params, results): + self.assertEqual(actual, expected) - def test_empty_strings_and_nones(self): - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) - s.row_factory = dict_factory + # verify data with prepared statement query + select = s.prepare("SELECT {0} FROM allcoltypes WHERE zz=?".format(columns_string)) + results = s.execute(select.bind([0]))[0] + for expected, actual in zip(params, results): + self.assertEqual(actual, expected) + + # verify data with with prepared statement, use dictionary with no explicit columns + s.row_factory = ordered_dict_factory + select = s.prepare("SELECT * FROM allcoltypes") + results = s.execute(select)[0] - self._create_all_types_table() + for expected, actual in zip(params, results.values()): + self.assertEqual(actual, expected) - columns_string = ','.join(self._col_names) - s.execute("INSERT INTO alltypes (key) VALUES (2)") - results = s.execute("SELECT %s FROM alltypes WHERE key=2" % columns_string) - self.assertTrue(all(x is None for x in results[0].values())) + c.shutdown() - prepared = s.prepare("SELECT %s FROM alltypes WHERE key=?" % columns_string) - results = s.execute(prepared.bind((2,))) - self.assertTrue(all(x is None for x in results[0].values())) + def test_can_insert_empty_strings_and_nulls(self): + """ + Test insertion of empty strings and null values + """ - # insert empty strings for string-like fields and fetch them - expected_values = {'text_col': '', 'ascii_col': '', 'varchar_col': '', 'listtext_col': [''], 'maptextint_col': OrderedMap({'': 3})} + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") + + # create table + alpha_type_list = ["zz int PRIMARY KEY"] + col_names = [] + start_index = ord('a') + for i, datatype in enumerate(PRIMITIVE_DATATYPES): + alpha_type_list.append("{0} {1}".format(chr(start_index + i), datatype)) + col_names.append(chr(start_index + i)) + + s.execute("CREATE TABLE alltypes ({0})".format(', '.join(alpha_type_list))) + + # verify all types initially null with simple statement + columns_string = ','.join(col_names) + s.execute("INSERT INTO alltypes (zz) VALUES (2)") + results = s.execute("SELECT {0} FROM alltypes WHERE zz=2".format(columns_string))[0] + self.assertTrue(all(x is None for x in results)) + + # verify all types initially null with prepared statement + select = s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)) + results = s.execute(select.bind([2]))[0] + self.assertTrue(all(x is None for x in results)) + + # insert empty strings for string-like fields + expected_values = {'j': '', 'a': '', 'n': ''} columns_string = ','.join(expected_values.keys()) placeholders = ','.join(["%s"] * len(expected_values)) - s.execute("INSERT INTO alltypes (key, %s) VALUES (3, %s)" % (columns_string, placeholders), expected_values.values()) - self.assertEqual(expected_values, - s.execute("SELECT %s FROM alltypes WHERE key=3" % columns_string)[0]) - self.assertEqual(expected_values, - s.execute(s.prepare("SELECT %s FROM alltypes WHERE key=?" % columns_string), (3,))[0]) + s.execute("INSERT INTO alltypes (zz, {0}) VALUES (3, {1})".format(columns_string, placeholders), expected_values.values()) + + # verify string types empty with simple statement + results = s.execute("SELECT {0} FROM alltypes WHERE zz=3".format(columns_string))[0] + for expected, actual in zip(expected_values.values(), results): + self.assertEqual(actual, expected) + + # verify string types empty with prepared statement + results = s.execute(s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)), [3])[0] + for expected, actual in zip(expected_values.values(), results): + self.assertEqual(actual, expected) # non-string types shouldn't accept empty strings - for col in ('bigint_col', 'boolean_col', 'decimal_col', 'double_col', - 'float_col', 'int_col', 'listtext_col', 'setint_col', - 'maptextint_col', 'uuid_col', 'timeuuid_col', 'varint_col'): - query = "INSERT INTO alltypes (key, %s) VALUES (4, %%s)" % col - try: + for col in ('b', 'd', 'e', 'f', 'g', 'i', 'l', 'm', 'o'): + query = "INSERT INTO alltypes (zz, {0}) VALUES (4, %s)".format(col) + with self.assertRaises(InvalidRequest): s.execute(query, ['']) - except InvalidRequest: - pass - else: - self.fail("Expected an InvalidRequest error when inserting an " - "emptry string for column %s" % (col, )) - - prepared = s.prepare("INSERT INTO alltypes (key, %s) VALUES (4, ?)" % col) - try: - s.execute(prepared, ['']) - except TypeError: - pass - else: - self.fail("Expected an InvalidRequest error when inserting an " - "emptry string for column %s with a prepared statement" % (col, )) - - # insert values for all columns - values = ['text', 'ascii', 1, True, Decimal('1.0'), 0.1, 0.1, - "1.2.3.4", 1, ['a'], set([1]), {'a': 1}, - datetime.now(), uuid4(), uuid1(), 'a', 1] - if self._cass_version >= (2, 1, 4): - values.append('2014-01-01') - values.append('01:02:03.456789012') - - columns_string = ','.join(self._col_names) - placeholders = ','.join(["%s"] * len(self._col_names)) - insert = "INSERT INTO alltypes (key, %s) VALUES (5, %s)" % (columns_string, placeholders) - s.execute(insert, values) + + insert = s.prepare("INSERT INTO alltypes (zz, {0}) VALUES (4, ?)".format(col)) + with self.assertRaises(TypeError): + s.execute(insert, ['']) + + # verify that Nones can be inserted and overwrites existing data + # create the input + params = [] + for datatype in PRIMITIVE_DATATYPES: + params.append((get_sample(datatype))) + + # insert the data + columns_string = ','.join(col_names) + placeholders = ','.join(["%s"] * len(col_names)) + simple_insert = "INSERT INTO alltypes (zz, {0}) VALUES (5, {1})".format(columns_string, placeholders) + s.execute(simple_insert, params) # then insert None, which should null them out - null_values = [None] * len(self._col_names) - s.execute(insert, null_values) + null_values = [None] * len(col_names) + s.execute(simple_insert, null_values) - select = "SELECT %s FROM alltypes WHERE key=5" % columns_string - results = s.execute(select) - self.assertEqual([], [(name, val) for (name, val) in results[0].items() if val is not None]) + # check via simple statement + query = "SELECT {0} FROM alltypes WHERE zz=5".format(columns_string) + results = s.execute(query)[0] + for col in results: + self.assertEqual(None, col) - prepared = s.prepare(select) - results = s.execute(prepared.bind(())) - self.assertEqual([], [(name, val) for (name, val) in results[0].items() if val is not None]) + # check via prepared statement + select = s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)) + results = s.execute(select.bind([5]))[0] + for col in results: + self.assertEqual(None, col) # do the same thing again, but use a prepared statement to insert the nulls - s.execute(insert, values) + s.execute(simple_insert, params) - placeholders = ','.join(["?"] * len(self._col_names)) - prepared = s.prepare("INSERT INTO alltypes (key, %s) VALUES (5, %s)" % (columns_string, placeholders)) - s.execute(prepared, null_values) + placeholders = ','.join(["?"] * len(col_names)) + insert = s.prepare("INSERT INTO alltypes (zz, {0}) VALUES (5, {1})".format(columns_string, placeholders)) + s.execute(insert, null_values) - results = s.execute(select) - self.assertEqual([], [(name, val) for (name, val) in results[0].items() if val is not None]) + results = s.execute(query)[0] + for col in results: + self.assertEqual(None, col) - prepared = s.prepare(select) - results = s.execute(prepared.bind(())) - self.assertEqual([], [(name, val) for (name, val) in results[0].items() if val is not None]) + results = s.execute(select.bind([5]))[0] + for col in results: + self.assertEqual(None, col) s.shutdown() - def test_empty_values(self): - s = self._session + def test_can_insert_empty_values_for_int32(self): + """ + Ensure Int32Type supports empty values + """ + + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") + s.execute("CREATE TABLE empty_values (a text PRIMARY KEY, b int)") s.execute("INSERT INTO empty_values (a, b) VALUES ('a', blobAsInt(0x))") try: @@ -364,8 +365,13 @@ def test_empty_values(self): finally: Int32Type.support_empty_values = False - def test_timezone_aware_datetimes(self): - """ Ensure timezone-aware datetimes are converted to timestamps correctly """ + c.shutdown() + + def test_timezone_aware_datetimes_are_timestamps(self): + """ + Ensure timezone-aware datetimes are converted to timestamps correctly + """ + try: import pytz except ImportError as exc: @@ -375,22 +381,25 @@ def test_timezone_aware_datetimes(self): eastern_tz = pytz.timezone('US/Eastern') eastern_tz.localize(dt) - s = self._session + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") s.execute("CREATE TABLE tz_aware (a ascii PRIMARY KEY, b timestamp)") # test non-prepared statement - s.execute("INSERT INTO tz_aware (a, b) VALUES ('key1', %s)", parameters=(dt,)) + s.execute("INSERT INTO tz_aware (a, b) VALUES ('key1', %s)", [dt]) result = s.execute("SELECT b FROM tz_aware WHERE a='key1'")[0].b self.assertEqual(dt.utctimetuple(), result.utctimetuple()) # test prepared statement - prepared = s.prepare("INSERT INTO tz_aware (a, b) VALUES ('key2', ?)") - s.execute(prepared, parameters=(dt,)) + insert = s.prepare("INSERT INTO tz_aware (a, b) VALUES ('key2', ?)") + s.execute(insert.bind([dt])) result = s.execute("SELECT b FROM tz_aware WHERE a='key2'")[0].b self.assertEqual(dt.utctimetuple(), result.utctimetuple()) - def test_tuple_type(self): + c.shutdown() + + def test_can_insert_tuples(self): """ Basic test of tuple functionality """ @@ -398,8 +407,8 @@ def test_tuple_type(self): if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") # use this encoder in order to insert tuples s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple @@ -439,9 +448,9 @@ def test_tuple_type(self): self.assertEqual(partial_result, s.execute(prepared, (4,))[0].b) self.assertEqual(subpartial_result, s.execute(prepared, (5,))[0].b) - s.shutdown() + c.shutdown() - def test_tuple_type_varying_lengths(self): + def test_can_insert_tuples_with_varying_lengths(self): """ Test tuple types of lengths of 1, 2, 3, and 384 to ensure edge cases work as expected. @@ -450,8 +459,8 @@ def test_tuple_type_varying_lengths(self): if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") # set the row_factory to dict_factory for programmatic access # set the encoder for tuples for the ability to write tuples @@ -479,9 +488,9 @@ def test_tuple_type_varying_lengths(self): result = s.execute("SELECT v_%s FROM tuple_lengths WHERE k=0", (i,))[0] self.assertEqual(tuple(created_tuple), result['v_%s' % i]) - s.shutdown() + c.shutdown() - def test_tuple_primitive_subtypes(self): + def test_can_insert_tuples_all_primitive_datatypes(self): """ Ensure tuple subtypes are appropriately handled. """ @@ -489,28 +498,28 @@ def test_tuple_primitive_subtypes(self): if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple s.execute("CREATE TABLE tuple_primitive (" "k int PRIMARY KEY, " - "v frozen>)" % ','.join(DATA_TYPE_PRIMITIVES)) + "v frozen>)" % ','.join(PRIMITIVE_DATATYPES)) - for i in range(len(DATA_TYPE_PRIMITIVES)): + for i in range(len(PRIMITIVE_DATATYPES)): # create tuples to be written and ensure they match with the expected response # responses have trailing None values for every element that has not been written - created_tuple = [get_sample(DATA_TYPE_PRIMITIVES[j]) for j in range(i + 1)] - response_tuple = tuple(created_tuple + [None for j in range(len(DATA_TYPE_PRIMITIVES) - i - 1)]) + created_tuple = [get_sample(PRIMITIVE_DATATYPES[j]) for j in range(i + 1)] + response_tuple = tuple(created_tuple + [None for j in range(len(PRIMITIVE_DATATYPES) - i - 1)]) written_tuple = tuple(created_tuple) s.execute("INSERT INTO tuple_primitive (k, v) VALUES (%s, %s)", (i, written_tuple)) result = s.execute("SELECT v FROM tuple_primitive WHERE k=%s", (i,))[0] self.assertEqual(response_tuple, result.v) - s.shutdown() + c.shutdown() - def test_tuple_non_primitive_subtypes(self): + def test_can_insert_tuples_all_collection_datatypes(self): """ Ensure tuple subtypes are appropriately handled for maps, sets, and lists. """ @@ -518,8 +527,8 @@ def test_tuple_non_primitive_subtypes(self): if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") # set the row_factory to dict_factory for programmatic access # set the encoder for tuples for the ability to write tuples @@ -529,15 +538,15 @@ def test_tuple_non_primitive_subtypes(self): values = [] # create list values - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: values.append('v_{} frozen>>'.format(len(values), datatype)) # create set values - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: values.append('v_{} frozen>>'.format(len(values), datatype)) # create map values - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: datatype_1 = datatype_2 = datatype if datatype == 'blob': # unhashable type: 'bytearray' @@ -545,9 +554,9 @@ def test_tuple_non_primitive_subtypes(self): values.append('v_{} frozen>>'.format(len(values), datatype_1, datatype_2)) # make sure we're testing all non primitive data types in the future - if set(DATA_TYPE_NON_PRIMITIVE_NAMES) != set(['tuple', 'list', 'map', 'set']): + if set(COLLECTION_TYPES) != set(['tuple', 'list', 'map', 'set']): raise NotImplemented('Missing datatype not implemented: {}'.format( - set(DATA_TYPE_NON_PRIMITIVE_NAMES) - set(['tuple', 'list', 'map', 'set']) + set(COLLECTION_TYPES) - set(['tuple', 'list', 'map', 'set']) )) # create table @@ -557,7 +566,7 @@ def test_tuple_non_primitive_subtypes(self): i = 0 # test tuple> - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: created_tuple = tuple([[get_sample(datatype)]]) s.execute("INSERT INTO tuple_non_primative (k, v_%s) VALUES (0, %s)", (i, created_tuple)) @@ -566,7 +575,7 @@ def test_tuple_non_primitive_subtypes(self): i += 1 # test tuple> - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: created_tuple = tuple([sortedset([get_sample(datatype)])]) s.execute("INSERT INTO tuple_non_primative (k, v_%s) VALUES (0, %s)", (i, created_tuple)) @@ -575,7 +584,7 @@ def test_tuple_non_primitive_subtypes(self): i += 1 # test tuple> - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: if datatype == 'blob': # unhashable type: 'bytearray' created_tuple = tuple([{get_sample('ascii'): get_sample(datatype)}]) @@ -587,7 +596,7 @@ def test_tuple_non_primitive_subtypes(self): result = s.execute("SELECT v_%s FROM tuple_non_primative WHERE k=0", (i,))[0] self.assertEqual(created_tuple, result['v_%s' % i]) i += 1 - s.shutdown() + c.shutdown() def nested_tuples_schema_helper(self, depth): """ @@ -609,7 +618,7 @@ def nested_tuples_creator_helper(self, depth): else: return (self.nested_tuples_creator_helper(depth - 1), ) - def test_nested_tuples(self): + def test_can_insert_nested_tuples(self): """ Ensure nested are appropriately handled. """ @@ -617,8 +626,8 @@ def test_nested_tuples(self): if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") # set the row_factory to dict_factory for programmatic access # set the encoder for tuples for the ability to write tuples @@ -647,16 +656,18 @@ def test_nested_tuples(self): # verify tuple was written and read correctly result = s.execute("SELECT v_%s FROM nested_tuples WHERE k=%s", (i, i))[0] self.assertEqual(created_tuple, result['v_%s' % i]) - s.shutdown() + c.shutdown() - def test_tuples_with_nulls(self): + def test_can_insert_tuples_with_nulls(self): """ Test tuples with null and empty string fields. """ + if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - s = self._session + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") s.execute("CREATE TABLE tuples_nulls (k int PRIMARY KEY, t frozen>)") @@ -675,75 +686,29 @@ def test_tuples_with_nulls(self): self.assertEqual(('', None, None, b''), result[0].t) self.assertEqual(('', None, None, b''), s.execute(read)[0].t) - def test_unicode_query_string(self): - s = self._session + c.shutdown() - query = u"SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = 'ef\u2052ef' AND columnfamily_name = %s" - s.execute(query, (u"fe\u2051fe",)) - - def insert_select_column(self, session, table_name, column_name, value): - insert = session.prepare("INSERT INTO %s (k, %s) VALUES (?, ?)" % (table_name, column_name)) - session.execute(insert, (0, value)) - result = session.execute("SELECT %s FROM %s WHERE k=%%s" % (column_name, table_name), (0,))[0][0] - self.assertEqual(result, value) - - def test_nested_collections(self): + def test_can_insert_unicode_query_string(self): + """ + Test to ensure unicode strings can be used in a query + """ - if self._cass_version < (2, 1, 3): - raise unittest.SkipTest("Support for nested collections was introduced in Cassandra 2.1.3") + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") - if PROTOCOL_VERSION < 3: - raise unittest.SkipTest("Protocol version > 3 required for nested collections") + query = u"SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = 'ef\u2052ef' AND columnfamily_name = %s" + s.execute(query, (u"fe\u2051fe",)) - name = self._testMethodName + c.shutdown() - s = self._session.cluster.connect() - s.set_keyspace(self._session.keyspace) - s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple + def test_can_read_composite_type(self): + """ + Test to ensure that CompositeTypes can be used in a query + """ - s.execute(""" - CREATE TYPE %s ( - m frozen>, - t tuple, - l frozen>, - s frozen> - )""" % name) - s.execute(""" - CREATE TYPE %s_nested ( - m frozen>, - t tuple, - l frozen>, - s frozen>, - u frozen<%s> - )""" % (name, name)) - s.execute(""" - CREATE TABLE %s ( - k int PRIMARY KEY, - map_map map>, frozen>>, - map_set map>, frozen>>, - map_list map>, frozen>>, - map_tuple map>, frozen>>, - map_udt map, frozen<%s>>, - )""" % (name, name, name)) - - validate = partial(self.insert_select_column, s, name) - validate('map_map', OrderedMap([({1: 1, 2: 2}, {3: 3, 4: 4}), ({5: 5, 6: 6}, {7: 7, 8: 8})])) - validate('map_set', OrderedMap([(set((1, 2)), set((3, 4))), (set((5, 6)), set((7, 8)))])) - validate('map_list', OrderedMap([([1, 2], [3, 4]), ([5, 6], [7, 8])])) - validate('map_tuple', OrderedMap([((1, 2), (3,)), ((4, 5), (6,))])) - - value = nested_collection_udt({1: 'v1', 2: 'v2'}, (3, 'v3'), [4, 5, 6, 7], set((8, 9, 10))) - key = nested_collection_udt_nested(value.m, value.t, value.l, value.s, value) - key2 = nested_collection_udt_nested({3: 'v3'}, value.t, value.l, value.s, value) - validate('map_udt', OrderedMap([(key, value), (key2, value)])) - - s.execute("DROP TABLE %s" % (name)) - s.execute("DROP TYPE %s_nested" % (name)) - s.execute("DROP TYPE %s" % (name)) - s.shutdown() + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("typetests") - def test_reading_composite_type(self): - s = self._session s.execute(""" CREATE TABLE composites ( a int PRIMARY KEY, @@ -761,3 +726,5 @@ def test_reading_composite_type(self): result = s.execute("SELECT * FROM composites WHERE a = 0")[0] self.assertEqual(0, result.a) self.assertEqual(('abc',), result.b) + + c.shutdown() From e2dcc133bdf658b35e3a3ed922b886fc76952358 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Tue, 7 Apr 2015 17:02:53 -0700 Subject: [PATCH 0007/2431] Refactored UDTTests --- tests/integration/standard/test_udts.py | 472 ++++++++++++------------ 1 file changed, 238 insertions(+), 234 deletions(-) diff --git a/tests/integration/standard/test_udts.py b/tests/integration/standard/test_udts.py index acdb42a183..10b9dd390c 100644 --- a/tests/integration/standard/test_udts.py +++ b/tests/integration/standard/test_udts.py @@ -11,51 +11,68 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from cassandra.query import dict_factory try: import unittest2 as unittest except ImportError: - import unittest # noqa + import unittest # noqa import logging log = logging.getLogger(__name__) from collections import namedtuple +from functools import partial +from cassandra import InvalidRequest from cassandra.cluster import Cluster, UserTypeDoesNotExist +from cassandra.query import dict_factory +from cassandra.util import OrderedMap from tests.integration import get_server_versions, use_singledc, PROTOCOL_VERSION -from tests.integration.datatype_utils import get_sample, get_nonprim_sample,\ - DATA_TYPE_PRIMITIVES, DATA_TYPE_NON_PRIMITIVE_NAMES +from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, COLLECTION_TYPES, \ + get_sample, get_collection_sample + +nested_collection_udt = namedtuple('nested_collection_udt', ['m', 't', 'l', 's']) +nested_collection_udt_nested = namedtuple('nested_collection_udt_nested', ['m', 't', 'l', 's', 'u']) def setup_module(): use_singledc() + update_datatypes() -class TypeTests(unittest.TestCase): +class UDTTests(unittest.TestCase): def setUp(self): - if PROTOCOL_VERSION < 3: - raise unittest.SkipTest("v3 protocol is required for UDT tests") - self._cass_version, self._cql_version = get_server_versions() - def test_unprepared_registered_udts(self): + if self._cass_version < (2, 1, 0): + raise unittest.SkipTest("User Defined Types were introduced in Cassandra 2.1") + + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + self.session.execute("CREATE KEYSPACE udttests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") + self.cluster.shutdown() + + def tearDown(self): + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + self.session.execute("DROP KEYSPACE udttests") + self.cluster.shutdown() + + def test_can_insert_unprepared_registered_udts(self): + """ + Test the insertion of unprepared, registered UDTs + """ + c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() + s = c.connect("udttests") - s.execute(""" - CREATE KEYSPACE udt_test_unprepared_registered - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } - """) - s.set_keyspace("udt_test_unprepared_registered") s.execute("CREATE TYPE user (age int, name text)") s.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen)") User = namedtuple('user', ('age', 'name')) - c.register_user_type("udt_test_unprepared_registered", "user", User) + c.register_user_type("udttests", "user", User) s.execute("INSERT INTO mytable (a, b) VALUES (%s, %s)", (0, User(42, 'bob'))) result = s.execute("SELECT b FROM mytable WHERE a=0") @@ -87,9 +104,10 @@ def test_unprepared_registered_udts(self): c.shutdown() - def test_register_before_connecting(self): - User1 = namedtuple('user', ('age', 'name')) - User2 = namedtuple('user', ('state', 'is_cool')) + def test_can_register_udt_before_connecting(self): + """ + Test the registration of UDTs before session creation + """ c = Cluster(protocol_version=PROTOCOL_VERSION) s = c.connect() @@ -113,6 +131,10 @@ def test_register_before_connecting(self): # now that types are defined, shutdown and re-create Cluster c.shutdown() c = Cluster(protocol_version=PROTOCOL_VERSION) + + User1 = namedtuple('user', ('age', 'name')) + User2 = namedtuple('user', ('state', 'is_cool')) + c.register_user_type("udt_test_register_before_connecting", "user", User1) c.register_user_type("udt_test_register_before_connecting2", "user", User2) @@ -139,15 +161,14 @@ def test_register_before_connecting(self): c.shutdown() - def test_prepared_unregistered_udts(self): + def test_can_insert_prepared_unregistered_udts(self): + """ + Test the insertion of prepared, unregistered UDTs + """ + c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() + s = c.connect("udttests") - s.execute(""" - CREATE KEYSPACE udt_test_prepared_unregistered - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } - """) - s.set_keyspace("udt_test_prepared_unregistered") s.execute("CREATE TYPE user (age int, name text)") s.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen)") @@ -184,18 +205,17 @@ def test_prepared_unregistered_udts(self): c.shutdown() - def test_prepared_registered_udts(self): + def test_can_insert_prepared_registered_udts(self): + """ + Test the insertion of prepared, registered UDTs + """ + c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() + s = c.connect("udttests") - s.execute(""" - CREATE KEYSPACE udt_test_prepared_registered - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } - """) - s.set_keyspace("udt_test_prepared_registered") s.execute("CREATE TYPE user (age int, name text)") User = namedtuple('user', ('age', 'name')) - c.register_user_type("udt_test_prepared_registered", "user", User) + c.register_user_type("udttests", "user", User) s.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen)") @@ -235,21 +255,17 @@ def test_prepared_registered_udts(self): c.shutdown() - def test_udts_with_nulls(self): + def test_can_insert_udts_with_nulls(self): """ - Test UDTs with null and empty string fields. + Test the insertion of UDTs with null and empty string fields """ + c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() + s = c.connect("udttests") - s.execute(""" - CREATE KEYSPACE test_udts_with_nulls - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } - """) - s.set_keyspace("test_udts_with_nulls") s.execute("CREATE TYPE user (a text, b int, c uuid, d blob)") User = namedtuple('user', ('a', 'b', 'c', 'd')) - c.register_user_type("test_udts_with_nulls", "user", User) + c.register_user_type("udttests", "user", User) s.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen)") @@ -270,38 +286,30 @@ def test_udts_with_nulls(self): c.shutdown() - def test_udt_sizes(self): + def test_can_insert_udts_with_varying_lengths(self): """ - Test for ensuring extra-lengthy udts are handled correctly. + Test for ensuring extra-lengthy udts are properly inserted """ - if self._cass_version < (2, 1, 0): - raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - - MAX_TEST_LENGTH = 16384 - EXTENDED_QUERY_TIMEOUT = 60 - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() + s = c.connect("udttests") - s.execute("""CREATE KEYSPACE test_udt_sizes - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""") - s.set_keyspace("test_udt_sizes") + MAX_TEST_LENGTH = 1024 # create the seed udt, increase timeout to avoid the query failure on slow systems s.execute("CREATE TYPE lengthy_udt ({})" .format(', '.join(['v_{} int'.format(i) - for i in range(MAX_TEST_LENGTH)])), timeout=EXTENDED_QUERY_TIMEOUT) + for i in range(MAX_TEST_LENGTH)]))) # create a table with multiple sizes of nested udts # no need for all nested types, only a spot checked few and the largest one s.execute("CREATE TABLE mytable (" "k int PRIMARY KEY, " - "v frozen)", timeout=EXTENDED_QUERY_TIMEOUT) + "v frozen)") # create and register the seed udt type udt = namedtuple('lengthy_udt', tuple(['v_{}'.format(i) for i in range(MAX_TEST_LENGTH)])) - c.register_user_type("test_udt_sizes", "lengthy_udt", udt) + c.register_user_type("udttests", "lengthy_udt", udt) # verify inserts and reads for i in (0, 1, 2, 3, MAX_TEST_LENGTH): @@ -313,245 +321,189 @@ def test_udt_sizes(self): s.execute("INSERT INTO mytable (k, v) VALUES (0, %s)", (created_udt,)) # verify udt was written and read correctly, increase timeout to avoid the query failure on slow systems - result = s.execute("SELECT v FROM mytable WHERE k=0", timeout=EXTENDED_QUERY_TIMEOUT)[0] + result = s.execute("SELECT v FROM mytable WHERE k=0")[0] self.assertEqual(created_udt, result.v) c.shutdown() - def nested_udt_helper(self, udts, i): - """ - Helper for creating nested udts. - """ + def nested_udt_schema_helper(self, session, MAX_NESTING_DEPTH): + # create the seed udt + session.execute("CREATE TYPE depth_0 (age int, name text)") + + # create the nested udts + for i in range(MAX_NESTING_DEPTH): + session.execute("CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) + # create a table with multiple sizes of nested udts + # no need for all nested types, only a spot checked few and the largest one + session.execute("CREATE TABLE mytable (" + "k int PRIMARY KEY, " + "v_0 frozen, " + "v_1 frozen, " + "v_2 frozen, " + "v_3 frozen, " + "v_{0} frozen)".format(MAX_NESTING_DEPTH)) + + def nested_udt_creation_helper(self, udts, i): if i == 0: return udts[0](42, 'Bob') else: - return udts[i](self.nested_udt_helper(udts, i - 1)) - - def test_nested_registered_udts(self): - """ - Test for ensuring nested udts are handled correctly. - """ + return udts[i](self.nested_udt_creation_helper(udts, i - 1)) - if self._cass_version < (2, 1, 0): - raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") + def nested_udt_verification_helper(self, session, MAX_NESTING_DEPTH, udts): + for i in (0, 1, 2, 3, MAX_NESTING_DEPTH): + # create udt + udt = self.nested_udt_creation_helper(udts, i) - MAX_NESTING_DEPTH = 16 + # write udt via simple statement + session.execute("INSERT INTO mytable (k, v_%s) VALUES (0, %s)", [i, udt]) - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() + # verify udt was written and read correctly + result = session.execute("SELECT v_{0} FROM mytable WHERE k=0".format(i))[0] + self.assertEqual(udt, result["v_{0}".format(i)]) - # set the row_factory to dict_factory for programmatically accessing values - s.row_factory = dict_factory + # write udt via prepared statement + insert = session.prepare("INSERT INTO mytable (k, v_{0}) VALUES (1, ?)".format(i)) + session.execute(insert, [udt]) - s.execute("""CREATE KEYSPACE test_nested_registered_udts - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""") - s.set_keyspace("test_nested_registered_udts") + # verify udt was written and read correctly + result = session.execute("SELECT v_{0} FROM mytable WHERE k=1".format(i))[0] + self.assertEqual(udt, result["v_{0}".format(i)]) - # create the seed udt - s.execute("CREATE TYPE depth_0 (age int, name text)") + def test_can_insert_nested_registered_udts(self): + """ + Test for ensuring nested registered udts are properly inserted + """ - # create the nested udts - for i in range(MAX_NESTING_DEPTH): - s.execute("CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("udttests") + s.row_factory = dict_factory - # create a table with multiple sizes of nested udts - # no need for all nested types, only a spot checked few and the largest one - s.execute("CREATE TABLE mytable (" - "k int PRIMARY KEY, " - "v_0 frozen, " - "v_1 frozen, " - "v_2 frozen, " - "v_3 frozen, " - "v_{0} frozen)".format(MAX_NESTING_DEPTH)) + MAX_NESTING_DEPTH = 16 - # create the udt container - udts = [] + # create the schema + self.nested_udt_schema_helper(s, MAX_NESTING_DEPTH) # create and register the seed udt type + udts = [] udt = namedtuple('depth_0', ('age', 'name')) udts.append(udt) - c.register_user_type("test_nested_registered_udts", "depth_0", udts[0]) + c.register_user_type("udttests", "depth_0", udts[0]) # create and register the nested udt types for i in range(MAX_NESTING_DEPTH): udt = namedtuple('depth_{}'.format(i + 1), ('value')) udts.append(udt) - c.register_user_type("test_nested_registered_udts", "depth_{}".format(i + 1), udts[i + 1]) + c.register_user_type("udttests", "depth_{}".format(i + 1), udts[i + 1]) - # verify inserts and reads - for i in (0, 1, 2, 3, MAX_NESTING_DEPTH): - # create udt - udt = self.nested_udt_helper(udts, i) - - # write udt - s.execute("INSERT INTO mytable (k, v_%s) VALUES (0, %s)", (i, udt)) - - # verify udt was written and read correctly - result = s.execute("SELECT v_%s FROM mytable WHERE k=0", (i,))[0] - self.assertEqual(udt, result['v_%s' % i]) + # insert udts and verify inserts with reads + self.nested_udt_verification_helper(s, MAX_NESTING_DEPTH, udts) c.shutdown() - def test_nested_unregistered_udts(self): + def test_can_insert_nested_unregistered_udts(self): """ - Test for ensuring nested unregistered udts are handled correctly. + Test for ensuring nested unregistered udts are properly inserted """ - if self._cass_version < (2, 1, 0): - raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - - MAX_NESTING_DEPTH = 16 c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() - - # set the row_factory to dict_factory for programmatically accessing values + s = c.connect("udttests") s.row_factory = dict_factory - s.execute("""CREATE KEYSPACE test_nested_unregistered_udts - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""") - s.set_keyspace("test_nested_unregistered_udts") - - # create the seed udt - s.execute("CREATE TYPE depth_0 (age int, name text)") - - # create the nested udts - for i in range(MAX_NESTING_DEPTH): - s.execute("CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) + MAX_NESTING_DEPTH = 16 - # create a table with multiple sizes of nested udts - # no need for all nested types, only a spot checked few and the largest one - s.execute("CREATE TABLE mytable (" - "k int PRIMARY KEY, " - "v_0 frozen, " - "v_1 frozen, " - "v_2 frozen, " - "v_3 frozen, " - "v_{0} frozen)".format(MAX_NESTING_DEPTH)) + # create the schema + self.nested_udt_schema_helper(s, MAX_NESTING_DEPTH) - # create the udt container + # create the seed udt type udts = [] - - # create and register the seed udt type udt = namedtuple('depth_0', ('age', 'name')) udts.append(udt) - # create and register the nested udt types + # create the nested udt types for i in range(MAX_NESTING_DEPTH): udt = namedtuple('depth_{}'.format(i + 1), ('value')) udts.append(udt) - # verify inserts and reads + # insert udts via prepared statements and verify inserts with reads for i in (0, 1, 2, 3, MAX_NESTING_DEPTH): # create udt - udt = self.nested_udt_helper(udts, i) + udt = self.nested_udt_creation_helper(udts, i) # write udt insert = s.prepare("INSERT INTO mytable (k, v_{0}) VALUES (0, ?)".format(i)) - s.execute(insert, (udt,)) + s.execute(insert, [udt]) # verify udt was written and read correctly - result = s.execute("SELECT v_%s FROM mytable WHERE k=0", (i,))[0] - self.assertEqual(udt, result['v_%s' % i]) + result = s.execute("SELECT v_{0} FROM mytable WHERE k=0".format(i))[0] + self.assertEqual(udt, result["v_{0}".format(i)]) c.shutdown() - def test_nested_registered_udts_with_different_namedtuples(self): + def test_can_insert_nested_registered_udts_with_different_namedtuples(self): """ - Test for ensuring nested udts are handled correctly when the + Test for ensuring nested udts are inserted correctly when the created namedtuples are use names that are different the cql type. - - Future improvement: optimize these three related tests using a single - helper method to cut down on code repetition. """ - if self._cass_version < (2, 1, 0): - raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - - MAX_NESTING_DEPTH = 16 - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() - - # set the row_factory to dict_factory for programmatically accessing values + s = c.connect("udttests") s.row_factory = dict_factory - s.execute("""CREATE KEYSPACE different_namedtuples - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}""") - s.set_keyspace("different_namedtuples") - - # create the seed udt - s.execute("CREATE TYPE depth_0 (age int, name text)") - - # create the nested udts - for i in range(MAX_NESTING_DEPTH): - s.execute("CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) - - # create a table with multiple sizes of nested udts - # no need for all nested types, only a spot checked few and the largest one - s.execute("CREATE TABLE mytable (" - "k int PRIMARY KEY, " - "v_0 frozen, " - "v_1 frozen, " - "v_2 frozen, " - "v_3 frozen, " - "v_{0} frozen)".format(MAX_NESTING_DEPTH)) + MAX_NESTING_DEPTH = 16 - # create the udt container - udts = [] + # create the schema + self.nested_udt_schema_helper(s, MAX_NESTING_DEPTH) # create and register the seed udt type + udts = [] udt = namedtuple('level_0', ('age', 'name')) udts.append(udt) - c.register_user_type("different_namedtuples", "depth_0", udts[0]) + c.register_user_type("udttests", "depth_0", udts[0]) # create and register the nested udt types for i in range(MAX_NESTING_DEPTH): udt = namedtuple('level_{}'.format(i + 1), ('value')) udts.append(udt) - c.register_user_type("different_namedtuples", "depth_{}".format(i + 1), udts[i + 1]) - - # verify inserts and reads - for i in (0, 1, 2, 3, MAX_NESTING_DEPTH): - # create udt - udt = self.nested_udt_helper(udts, i) + c.register_user_type("udttests", "depth_{}".format(i + 1), udts[i + 1]) - # write udt - s.execute("INSERT INTO mytable (k, v_%s) VALUES (0, %s)", (i, udt)) - - # verify udt was written and read correctly - result = s.execute("SELECT v_%s FROM mytable WHERE k=0", (i,))[0] - self.assertEqual(udt, result['v_%s' % i]) + # insert udts and verify inserts with reads + self.nested_udt_verification_helper(s, MAX_NESTING_DEPTH, udts) c.shutdown() - def test_non_existing_types(self): + def test_raise_error_on_nonexisting_udts(self): + """ + Test for ensuring that an error is raised for operating on a nonexisting udt or an invalid keyspace + """ + c = Cluster(protocol_version=PROTOCOL_VERSION) - c.connect() + s = c.connect("udttests") User = namedtuple('user', ('age', 'name')) - self.assertRaises(UserTypeDoesNotExist, c.register_user_type, "some_bad_keyspace", "user", User) - self.assertRaises(UserTypeDoesNotExist, c.register_user_type, "system", "user", User) + + with self.assertRaises(UserTypeDoesNotExist): + c.register_user_type("some_bad_keyspace", "user", User) + + with self.assertRaises(UserTypeDoesNotExist): + c.register_user_type("system", "user", User) + + with self.assertRaises(InvalidRequest): + s.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen)") c.shutdown() - def test_primitive_datatypes(self): + def test_can_insert_udt_all_datatypes(self): """ - Test for inserting various types of DATA_TYPE_PRIMITIVES into UDT's + Test for inserting various types of PRIMITIVE_DATATYPES into UDT's """ - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() - # create keyspace - s.execute(""" - CREATE KEYSPACE test_primitive_datatypes - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } - """) - s.set_keyspace("test_primitive_datatypes") + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("udttests") # create UDT alpha_type_list = [] start_index = ord('a') - for i, datatype in enumerate(DATA_TYPE_PRIMITIVES): + for i, datatype in enumerate(PRIMITIVE_DATATYPES): alpha_type_list.append("{0} {1}".format(chr(start_index + i), datatype)) s.execute(""" @@ -563,14 +515,14 @@ def test_primitive_datatypes(self): # register UDT alphabet_list = [] - for i in range(ord('a'), ord('a') + len(DATA_TYPE_PRIMITIVES)): + for i in range(ord('a'), ord('a') + len(PRIMITIVE_DATATYPES)): alphabet_list.append('{}'.format(chr(i))) Alldatatypes = namedtuple("alldatatypes", alphabet_list) - c.register_user_type("test_primitive_datatypes", "alldatatypes", Alldatatypes) + c.register_user_type("udttests", "alldatatypes", Alldatatypes) # insert UDT data params = [] - for datatype in DATA_TYPE_PRIMITIVES: + for datatype in PRIMITIVE_DATATYPES: params.append((get_sample(datatype))) insert = s.prepare("INSERT INTO mytable (a, b) VALUES (?, ?)") @@ -586,34 +538,28 @@ def test_primitive_datatypes(self): c.shutdown() - def test_nonprimitive_datatypes(self): + def test_can_insert_udt_all_collection_datatypes(self): """ - Test for inserting various types of DATA_TYPE_NON_PRIMITIVE into UDT's + Test for inserting various types of COLLECTION_TYPES into UDT's """ - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect() - # create keyspace - s.execute(""" - CREATE KEYSPACE test_nonprimitive_datatypes - WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } - """) - s.set_keyspace("test_nonprimitive_datatypes") + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("udttests") # create UDT alpha_type_list = [] start_index = ord('a') - for i, nonprim_datatype in enumerate(DATA_TYPE_NON_PRIMITIVE_NAMES): - for j, datatype in enumerate(DATA_TYPE_PRIMITIVES): - if nonprim_datatype == "map": + for i, collection_type in enumerate(COLLECTION_TYPES): + for j, datatype in enumerate(PRIMITIVE_DATATYPES): + if collection_type == "map": type_string = "{0}_{1} {2}<{3}, {3}>".format(chr(start_index + i), chr(start_index + j), - nonprim_datatype, datatype) - elif nonprim_datatype == "tuple": + collection_type, datatype) + elif collection_type == "tuple": type_string = "{0}_{1} frozen<{2}<{3}>>".format(chr(start_index + i), chr(start_index + j), - nonprim_datatype, datatype) + collection_type, datatype) else: type_string = "{0}_{1} {2}<{3}>".format(chr(start_index + i), chr(start_index + j), - nonprim_datatype, datatype) + collection_type, datatype) alpha_type_list.append(type_string) s.execute(""" @@ -625,18 +571,18 @@ def test_nonprimitive_datatypes(self): # register UDT alphabet_list = [] - for i in range(ord('a'), ord('a') + len(DATA_TYPE_NON_PRIMITIVE_NAMES)): - for j in range(ord('a'), ord('a') + len(DATA_TYPE_PRIMITIVES)): + for i in range(ord('a'), ord('a') + len(COLLECTION_TYPES)): + for j in range(ord('a'), ord('a') + len(PRIMITIVE_DATATYPES)): alphabet_list.append('{0}_{1}'.format(chr(i), chr(j))) Alldatatypes = namedtuple("alldatatypes", alphabet_list) - c.register_user_type("test_nonprimitive_datatypes", "alldatatypes", Alldatatypes) + c.register_user_type("udttests", "alldatatypes", Alldatatypes) # insert UDT data params = [] - for nonprim_datatype in DATA_TYPE_NON_PRIMITIVE_NAMES: - for datatype in DATA_TYPE_PRIMITIVES: - params.append((get_nonprim_sample(nonprim_datatype, datatype))) + for collection_type in COLLECTION_TYPES: + for datatype in PRIMITIVE_DATATYPES: + params.append((get_collection_sample(collection_type, datatype))) insert = s.prepare("INSERT INTO mytable (a, b) VALUES (?, ?)") s.execute(insert, (0, Alldatatypes(*params))) @@ -650,3 +596,61 @@ def test_nonprimitive_datatypes(self): self.assertEqual(expected, actual) c.shutdown() + + def insert_select_column(self, session, table_name, column_name, value): + insert = session.prepare("INSERT INTO %s (k, %s) VALUES (?, ?)" % (table_name, column_name)) + session.execute(insert, (0, value)) + result = session.execute("SELECT %s FROM %s WHERE k=%%s" % (column_name, table_name), (0,))[0][0] + self.assertEqual(result, value) + + def test_can_insert_nested_collections(self): + """ + Test for inserting various types of nested COLLECTION_TYPES into tables and UDTs + """ + + if self._cass_version < (2, 1, 3): + raise unittest.SkipTest("Support for nested collections was introduced in Cassandra 2.1.3") + + c = Cluster(protocol_version=PROTOCOL_VERSION) + s = c.connect("udttests") + s.encoder.mapping[tuple] = s.encoder.cql_encode_tuple + + name = self._testMethodName + + s.execute(""" + CREATE TYPE %s ( + m frozen>, + t tuple, + l frozen>, + s frozen> + )""" % name) + s.execute(""" + CREATE TYPE %s_nested ( + m frozen>, + t tuple, + l frozen>, + s frozen>, + u frozen<%s> + )""" % (name, name)) + s.execute(""" + CREATE TABLE %s ( + k int PRIMARY KEY, + map_map map>, frozen>>, + map_set map>, frozen>>, + map_list map>, frozen>>, + map_tuple map>, frozen>>, + map_udt map, frozen<%s>>, + )""" % (name, name, name)) + + validate = partial(self.insert_select_column, s, name) + validate('map_map', OrderedMap([({1: 1, 2: 2}, {3: 3, 4: 4}), ({5: 5, 6: 6}, {7: 7, 8: 8})])) + validate('map_set', OrderedMap([(set((1, 2)), set((3, 4))), (set((5, 6)), set((7, 8)))])) + validate('map_list', OrderedMap([([1, 2], [3, 4]), ([5, 6], [7, 8])])) + validate('map_tuple', OrderedMap([((1, 2), (3,)), ((4, 5), (6,))])) + + value = nested_collection_udt({1: 'v1', 2: 'v2'}, (3, 'v3'), [4, 5, 6, 7], set((8, 9, 10))) + key = nested_collection_udt_nested(value.m, value.t, value.l, value.s, value) + key2 = nested_collection_udt_nested({3: 'v3'}, value.t, value.l, value.s, value) + validate('map_udt', OrderedMap([(key, value), (key2, value)])) + + c.shutdown() \ No newline at end of file From e89a23a91b0c5de6b84c75884aaf61b65f68d6c9 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Tue, 7 Apr 2015 17:20:09 -0700 Subject: [PATCH 0008/2431] add pure-sasl as a test-requirement for SaslAuthenticatorTests --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index e4d2d3836b..8fc8dad04a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,3 +8,4 @@ unittest2 PyYAML pytz sure +pure-sasl From 79c89ed67c60aed3c6e2ff67fd8469eb93873fc6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Apr 2015 11:26:33 -0500 Subject: [PATCH 0009/2431] Add C* 3.0/v4 to integration test default protocol --- tests/integration/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index ce3f042bf6..3ea397f01e 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -118,7 +118,9 @@ def _tuple_version(version_string): log.info('Using Cassandra version: %s', CASSANDRA_VERSION) CCM_KWARGS['version'] = CASSANDRA_VERSION -if CASSANDRA_VERSION > '2.1': +if CASSANDRA_VERSION > '3.0': + default_protocol_version = 4 +elif CASSANDRA_VERSION > '2.1': default_protocol_version = 3 elif CASSANDRA_VERSION > '2.0': default_protocol_version = 2 From 37677993faee7ec4c31da651cf67b13a7b34db03 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Apr 2015 12:42:51 -0500 Subject: [PATCH 0010/2431] Add a test for prepared statements with no column meta --- .../standard/test_prepared_statements.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index a75e6bc3d2..9240996ce8 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -213,6 +213,31 @@ def test_none_values(self): cluster.shutdown() + def test_no_meta(self): + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + session = cluster.connect() + + prepared = session.prepare( + """ + INSERT INTO test3rf.test (k, v) VALUES (0, 0) + """) + + self.assertIsInstance(prepared, PreparedStatement) + bound = prepared.bind(None) + session.execute(bound) + + prepared = session.prepare( + """ + SELECT * FROM test3rf.test WHERE k=0 + """) + self.assertIsInstance(prepared, PreparedStatement) + + bound = prepared.bind(None) + results = session.execute(bound) + self.assertEqual(results[0].v, 0) + + cluster.shutdown() + def test_none_values_dicts(self): """ Ensure binding None is handled correctly with dict bindings From 5dfc8b688d22d6fd8c309aee8e253947f3f667db Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Apr 2015 15:30:49 -0500 Subject: [PATCH 0011/2431] Change R/W failure exception base to CoordinationFailure --- cassandra/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index c1e0bdd807..eacc29e09a 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -221,7 +221,7 @@ def __init__(self, message, write_type=None, **kwargs): self.write_type = write_type -class Failure(Exception): +class CoordinationFailure(Exception): """ Replicas sent a failure to the coordinator. """ @@ -252,12 +252,12 @@ def __init__(self, summary_message, consistency=None, required_responses=None, r repr({'consistency': consistency_value_to_name(consistency), 'required_responses': required_responses, 'received_responses': received_responses, - 'failures' : failures})) + 'failures': failures})) -class ReadFailure(Failure): +class ReadFailure(CoordinationFailure): """ - A subclass of :exc:`Failure` for read operations. + A subclass of :exc:`CoordinationFailure` for read operations. This indicates that the replicas sent a failure message to the coordinator. """ @@ -270,13 +270,13 @@ class ReadFailure(Failure): """ def __init__(self, message, data_retrieved=None, **kwargs): - Failure.__init__(self, message, **kwargs) + CoordinationFailure.__init__(self, message, **kwargs) self.data_retrieved = data_retrieved -class WriteFailure(Failure): +class WriteFailure(CoordinationFailure): """ - A subclass of :exc:`Failure` for write operations. + A subclass of :exc:`CoordinationFailure` for write operations. This indicates that the replicas sent a failure message to the coordinator. """ @@ -287,7 +287,7 @@ class WriteFailure(Failure): """ def __init__(self, message, write_type=None, **kwargs): - Failure.__init__(self, message, **kwargs) + CoordinationFailure.__init__(self, message, **kwargs) self.write_type = write_type From 275a2663aa9e466b517729324d32765d17eae1bd Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Apr 2015 15:31:40 -0500 Subject: [PATCH 0012/2431] Add FunctionFailure error message and exception --- cassandra/__init__.py | 27 +++++++++++++++++++++++++++ cassandra/protocol.py | 19 ++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index eacc29e09a..a9c7e21ad6 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -291,6 +291,33 @@ def __init__(self, message, write_type=None, **kwargs): self.write_type = write_type +class FunctionFailure(Exception): + """ + User Defined Function failed during execution + """ + + keyspace = None + """ + Keyspace of the function + """ + + function = None + """ + Name of the function + """ + + arg_types = None + """ + Argument types of the function + """ + + def __init__(self, summary_message, keyspace, function, arg_types): + self.keyspace = keyspace + self.function = function + self.arg_types = arg_types + Exception.__init__(self, summary_message) + + class AlreadyExists(Exception): """ An attempt was made to create a keyspace or table that already exists. diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 527aa262b2..bdf7ae2b81 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -22,7 +22,7 @@ import io from cassandra import (Unavailable, WriteTimeout, ReadTimeout, - WriteFailure, ReadFailure, + WriteFailure, ReadFailure, FunctionFailure, AlreadyExists, InvalidRequest, Unauthorized, UnsupportedOperation) from cassandra.marshal import (int32_pack, int32_unpack, uint16_pack, uint16_unpack, @@ -282,6 +282,22 @@ def to_exception(self): return ReadFailure(self.summary_msg(), **self.info) +class FunctionFailureMessage(RequestExecutionException): + summary = "User Defined Function failure" + error_code = 0x1400 + + @staticmethod + def recv_error_info(f): + return { + 'keyspace': read_string(f), + 'function': read_string(f), + 'arg_types': [read_string(f) for _ in range(read_short(f))], + } + + def to_exception(self): + return FunctionFailure(self.summary_msg(), **self.info) + + class WriteFailureMessage(RequestExecutionException): summary = "Replica(s) failed to execute write" error_code = 0x1500 @@ -299,6 +315,7 @@ def recv_error_info(f): def to_exception(self): return WriteFailure(self.summary_msg(), **self.info) + class SyntaxException(RequestValidationException): summary = 'Syntax error in CQL query' error_code = 0x2000 From d5626750b184047a9f2f50465fd6edbb3a1b77cc Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Apr 2015 16:47:56 -0500 Subject: [PATCH 0013/2431] Add new request exceptions to docs --- cassandra/__init__.py | 2 +- docs/api/cassandra.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index a9c7e21ad6..fe57b2f90e 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -308,7 +308,7 @@ class FunctionFailure(Exception): arg_types = None """ - Argument types of the function + List of argument type names of the function """ def __init__(self, summary_message, keyspace, function, arg_types): diff --git a/docs/api/cassandra.rst b/docs/api/cassandra.rst index 90d23d108e..fd0de0be0e 100644 --- a/docs/api/cassandra.rst +++ b/docs/api/cassandra.rst @@ -26,6 +26,15 @@ .. autoexception:: WriteTimeout() :members: +.. autoexception:: ReadFailure() + :members: + +.. autoexception:: WriteFailure() + :members: + +.. autoexception:: FunctionFailure() + :members: + .. autoexception:: AlreadyExists() :members: From 4136a42601ce453de51862ac9368b84e05b95c98 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Apr 2015 11:58:52 -0500 Subject: [PATCH 0014/2431] Populate QueryTrace.client for C* 3.0+ --- cassandra/query.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cassandra/query.py b/cassandra/query.py index fc71b8f62b..cbe021fe4e 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -761,6 +761,13 @@ class QueryTrace(object): A :class:`datetime.timedelta` measure of the duration of the query. """ + client = None + """ + The IP address of the client that issued this request + + This is only available when using Cassandra 3.0+ + """ + coordinator = None """ The IP address of the host that acted as coordinator for this request. @@ -829,6 +836,8 @@ def populate(self, max_wait=2.0): self.started_at = session_row.started_at self.coordinator = session_row.coordinator self.parameters = session_row.parameters + # since C* 3.0 + self.client = getattr(session_row, 'client', None) log.debug("Attempting to fetch trace events for trace ID: %s", self.trace_id) time_spent = time.time() - start From 96d4a485ada62247796857467cbdf7db6735d9a0 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Apr 2015 16:40:53 -0500 Subject: [PATCH 0015/2431] Retrive UDF metadata --- cassandra/cluster.py | 15 +++++- cassandra/metadata.py | 121 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 132 insertions(+), 4 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cb04152a5b..e14c882c4a 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1820,6 +1820,7 @@ class ControlConnection(object): _SELECT_COLUMN_FAMILIES = "SELECT * FROM system.schema_columnfamilies" _SELECT_COLUMNS = "SELECT * FROM system.schema_columns" _SELECT_USERTYPES = "SELECT * FROM system.schema_usertypes" + _SELECT_FUNCTIONS = "SELECT * FROM system.schema_functions" _SELECT_TRIGGERS = "SELECT * FROM system.schema_triggers" _SELECT_PEERS = "SELECT peer, data_center, rack, tokens, rpc_address, schema_version FROM system.peers" @@ -2088,12 +2089,14 @@ def _handle_results(success, result): QueryMessage(query=self._SELECT_COLUMN_FAMILIES, consistency_level=cl), QueryMessage(query=self._SELECT_COLUMNS, consistency_level=cl), QueryMessage(query=self._SELECT_USERTYPES, consistency_level=cl), + QueryMessage(query=self._SELECT_FUNCTIONS, consistency_level=cl), QueryMessage(query=self._SELECT_TRIGGERS, consistency_level=cl) ] responses = connection.wait_for_responses(*queries, timeout=self._timeout, fail_on_error=False) (ks_success, ks_result), (cf_success, cf_result), \ (col_success, col_result), (types_success, types_result), \ + (functions_success, functions_result), \ (trigger_success, triggers_result) = responses if ks_success: @@ -2135,8 +2138,18 @@ def _handle_results(success, result): else: raise types_result + # functions were introduced in Cassandra 3.0 + if functions_success: + functions_result = dict_factory(*functions_result.results) if functions_result.results else {} + else: + if isinstance(functions_result, InvalidRequest): + log.debug("[control connection] user functions table not found") + functions_result = {} + else: + raise functions_result + log.debug("[control connection] Fetched schema, rebuilding metadata") - self._cluster.metadata.rebuild_schema(ks_result, types_result, cf_result, col_result, triggers_result) + self._cluster.metadata.rebuild_schema(ks_result, types_result, functions_result, cf_result, col_result, triggers_result) return True def refresh_node_list_and_token_map(self, force_token_rebuild=False): diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 1f88d8f9e6..896b848b5a 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -19,9 +19,10 @@ import json import logging import re +import six +from six.moves import zip from threading import RLock import weakref -import six murmur3 = None try: @@ -88,7 +89,8 @@ def export_schema_as_string(self): """ return "\n".join(ks.export_as_string() for ks in self.keyspaces.values()) - def rebuild_schema(self, ks_results, type_results, cf_results, col_results, triggers_result): + def rebuild_schema(self, ks_results, type_results, function_results, + cf_results, col_results, triggers_result): """ Rebuild the view of the current schema from a fresh set of rows from the system schema tables. @@ -98,6 +100,7 @@ def rebuild_schema(self, ks_results, type_results, cf_results, col_results, trig cf_def_rows = defaultdict(list) col_def_rows = defaultdict(lambda: defaultdict(list)) usertype_rows = defaultdict(list) + fn_rows = defaultdict(list) trigger_rows = defaultdict(lambda: defaultdict(list)) for row in cf_results: @@ -111,6 +114,9 @@ def rebuild_schema(self, ks_results, type_results, cf_results, col_results, trig for row in type_results: usertype_rows[row["keyspace_name"]].append(row) + for row in function_results: + fn_rows[row["keyspace_name"]].append(row) + for row in triggers_result: ksname = row["keyspace_name"] cfname = row["columnfamily_name"] @@ -131,6 +137,10 @@ def rebuild_schema(self, ks_results, type_results, cf_results, col_results, trig usertype = self._build_usertype(keyspace_meta.name, usertype_row) keyspace_meta.user_types[usertype.name] = usertype + for fn_row in fn_rows.get(keyspace_meta.name, []): + fn = self._build_function(keyspace_meta.name, fn_row) + keyspace_meta.functions[fn.name] = fn + current_keyspaces.add(keyspace_meta.name) old_keyspace_meta = self.keyspaces.get(keyspace_meta.name, None) self.keyspaces[keyspace_meta.name] = keyspace_meta @@ -140,7 +150,7 @@ def rebuild_schema(self, ks_results, type_results, cf_results, col_results, trig self._keyspace_added(keyspace_meta.name) # remove not-just-added keyspaces - removed_keyspaces = [ksname for ksname in self.keyspaces.keys() + removed_keyspaces = [name for name in self.keyspaces.keys() if ksname not in current_keyspaces] self.keyspaces = dict((name, meta) for name, meta in self.keyspaces.items() if name in current_keyspaces) @@ -215,6 +225,14 @@ def _build_usertype(self, keyspace, usertype_row): return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], usertype_row['field_names'], type_classes) + def _build_function(self, keyspace, function_row): + return_type = types.lookup_casstype(function_row['return_type']) + return Function(function_row['keyspace_name'], function_row['function_name'], + function_row['signature'], function_row['argument_names'], + return_type, function_row['language'], function_row['body'], + function_row['is_deterministic'], function_row.get('called_on_null_input')) + # called_on_null_input is not yet merged + def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) @@ -740,12 +758,19 @@ class KeyspaceMetadata(object): .. versionadded:: 2.1.0 """ + functions = None + """ + A map from user-defined function names to instances of :class:`~cassandra.metadata..Function`. + + .. versionadded:: 3.0.0 + """ def __init__(self, name, durable_writes, strategy_class, strategy_options): self.name = name self.durable_writes = durable_writes self.replication_strategy = ReplicationStrategy.create(strategy_class, strategy_options) self.tables = {} self.user_types = {} + self.functions = {} def export_as_string(self): """ @@ -843,6 +868,96 @@ def as_cql_query(self, formatted=False): return ret +class Function(object): + """ + A user defined function, as created by ``CREATE FUNCTION`` statements. + + User-defined functions were introduced in Cassandra 3.0 + + .. versionadded:: 3.0.0 + """ + + keyspace = None + """ + The string name of the keyspace in which this function is defined + """ + + name = None + """ + The name of this function + """ + + signature = None + """ + An ordered list of the types for each argument to the function + """ + + arguemnt_names = None + """ + An ordered list of the names of each argument to the function + """ + + return_type = None + """ + Return type of the function + """ + + language = None + """ + Language of the function body + """ + + body = None + """ + Function body string + """ + + is_deterministic = None + """ + Flag indicating whether this function is deterministic + (required for functional indexes) + """ + + called_on_null_input = None + """ + Flag indicating whether this function should be called for rows with null values + (convenience function to avoid handling nulls explicitly if the result will just be null) + """ + + def __init__(self, keyspace, name, signature, argument_names, + return_type, language, body, is_deterministic, called_on_null_input): + self.keyspace = keyspace + self.name = name + self.signature = signature + self.argument_names = argument_names + self.return_type = return_type + self.language = language + self.body = body + self.is_deterministic = is_deterministic + self.called_on_null_input = called_on_null_input + + def as_cql_query(self, formatted=False): + """ + Returns a CQL query that can be used to recreate this function. + If `formatted` is set to :const:`True`, extra whitespace will + be added to make the query more readable. + """ + sep = '\n' if formatted else ' ' + keyspace = protect_name(self.keyspace) + name = protect_name(self.name) + arg_list = ', '.join(["%s %s" % (protect_name(n), t) + for n, t in zip(self.argument_names, self.signature)]) + determ = '' if self.is_deterministic else 'NON DETERMINISTIC ' + typ = self.return_type.cql_parameterized_type() + lang = self.language + body = protect_value(self.body) + + return "CREATE %(determ)sFUNCTION %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ + "RETURNS %(typ)s%(sep)s" \ + "LANGUAGE %(lang)s%(sep)s" \ + "AS %(body)s;" % locals() + + class TableMetadata(object): """ A representation of the schema for a single table. From 6b509718039635be9b8d3f4b3da76b4d540bda15 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Apr 2015 14:20:54 -0500 Subject: [PATCH 0016/2431] Handle function schema_change events; basic tests Possibly still some server-side issues to iron out --- cassandra/cluster.py | 34 +++++---- cassandra/metadata.py | 9 +++ cassandra/protocol.py | 4 +- tests/integration/standard/test_metadata.py | 80 +++++++++++++++++++++ 4 files changed, 113 insertions(+), 14 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index e14c882c4a..96eb4044c9 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1116,7 +1116,7 @@ def _ensure_core_connections(self): for pool in session._pools.values(): pool.ensure_core_connections() - def refresh_schema(self, keyspace=None, table=None, usertype=None, max_schema_agreement_wait=None): + def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, max_schema_agreement_wait=None): """ Synchronously refresh the schema metadata. @@ -1129,7 +1129,7 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, max_schema_ag An Exception is raised if schema refresh fails for any reason. """ - if not self.control_connection.refresh_schema(keyspace, table, usertype, max_schema_agreement_wait): + if not self.control_connection.refresh_schema(keyspace, table, usertype, function, max_schema_agreement_wait): raise Exception("Schema was not refreshed. See log for details.") def submit_schema_refresh(self, keyspace=None, table=None, usertype=None): @@ -2008,7 +2008,7 @@ def shutdown(self): self._connection.close() del self._connection - def refresh_schema(self, keyspace=None, table=None, usertype=None, + def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, schema_agreement_wait=None): if not self._meta_refresh_enabled: log.debug("[control connection] Skipping schema refresh because meta refresh is disabled") @@ -2016,7 +2016,7 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, try: if self._connection: - return self._refresh_schema(self._connection, keyspace, table, usertype, + return self._refresh_schema(self._connection, keyspace, table, usertype, function, schema_agreement_wait=schema_agreement_wait) except ReferenceError: pass # our weak reference to the Cluster is no good @@ -2025,7 +2025,7 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, self._signal_error() return False - def _refresh_schema(self, connection, keyspace=None, table=None, usertype=None, + def _refresh_schema(self, connection, keyspace=None, table=None, usertype=None, function=None, preloaded_results=None, schema_agreement_wait=None): if self._cluster.is_shutdown: return False @@ -2074,6 +2074,14 @@ def _handle_results(success, result): log.debug("[control connection] Fetched user type info for %s.%s, rebuilding metadata", keyspace, usertype) types_result = dict_factory(*types_result.results) if types_result.results else {} self._cluster.metadata.usertype_changed(keyspace, usertype, types_result) + elif function: + # user defined function within this keyspace changed + where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s'" % (keyspace, function) + functions_query = QueryMessage(query=self._SELECT_FUNCTIONS + where_clause, consistency_level=cl) + functions_result = connection.wait_for_response(functions_query) + log.debug("[control connection] Fetched user function info for %s.%s, rebuilding metadata", keyspace, function) + functions_result = dict_factory(*functions_result.results) if functions_result.results else {} + self._cluster.metadata.function_changed(keyspace, function, functions_result) elif keyspace: # only the keyspace itself changed (such as replication settings) where_clause = " WHERE keyspace_name = '%s'" % (keyspace,) @@ -2297,8 +2305,9 @@ def _handle_schema_change(self, event): keyspace = event.get('keyspace') table = event.get('table') usertype = event.get('type') + function = event.get('function', event.get('aggregate')) delay = random() * self._schema_event_refresh_window - self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype) + self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype, function) def wait_for_schema_agreement(self, connection=None, preloaded_results=None, wait_time=None): @@ -2527,19 +2536,19 @@ def _log_if_failed(self, future): exc_info=exc) -def refresh_schema_and_set_result(keyspace, table, usertype, control_conn, response_future): +def refresh_schema_and_set_result(keyspace, table, usertype, function, control_conn, response_future): try: if control_conn._meta_refresh_enabled: - log.debug("Refreshing schema in response to schema change. Keyspace: %s; Table: %s, Type: %s", - keyspace, table, usertype) - control_conn._refresh_schema(response_future._connection, keyspace, table, usertype) + log.debug("Refreshing schema in response to schema change. Keyspace: %s; Table: %s, Type: %s, Function: %s", + keyspace, table, usertype, function) + control_conn._refresh_schema(response_future._connection, keyspace, table, usertype, function) else: log.debug("Skipping schema refresh in response to schema change because meta refresh is disabled; " - "Keyspace: %s; Table: %s, Type: %s", keyspace, table, usertype) + "Keyspace: %s; Table: %s, Type: %s, Function: %s", keyspace, table, usertype, function) except Exception: log.exception("Exception refreshing schema in response to schema change:") response_future.session.submit( - control_conn.refresh_schema, keyspace, table, usertype) + control_conn.refresh_schema, keyspace, table, usertype, function) finally: response_future._set_final_result(None) @@ -2724,6 +2733,7 @@ def _set_result(self, response): response.results['keyspace'], response.results.get('table'), response.results.get('type'), + response.results.get('function', response.results.get('aggregate')), self.session.cluster.control_connection, self) else: diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 896b848b5a..d6cd8e5840 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -170,6 +170,7 @@ def keyspace_changed(self, keyspace, ks_results): if old_keyspace_meta: keyspace_meta.tables = old_keyspace_meta.tables keyspace_meta.user_types = old_keyspace_meta.user_types + keyspace_meta.functions = old_keyspace_meta.functions if (keyspace_meta.replication_strategy != old_keyspace_meta.replication_strategy): self._keyspace_updated(keyspace) else: @@ -183,6 +184,14 @@ def usertype_changed(self, keyspace, name, type_results): # the type was deleted self.keyspaces[keyspace].user_types.pop(name, None) + def function_changed(self, keyspace, name, function_results): + if function_results: + new_function = self._build_function(keyspace, function_results[0]) + self.keyspaces[keyspace].functions[name] = new_function + else: + # the function was deleted + self.keyspaces[keyspace].functions.pop(name, None) + def table_changed(self, keyspace, table, cf_results, col_results, triggers_result): try: keyspace_meta = self.keyspaces[keyspace] diff --git a/cassandra/protocol.py b/cassandra/protocol.py index d81a650731..922925a526 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -851,8 +851,8 @@ def recv_schema_change(cls, f, protocol_version): target = read_string(f) keyspace = read_string(f) if target != "KEYSPACE": - table_or_type = read_string(f) - return {'change_type': change_type, 'keyspace': keyspace, target.lower(): table_or_type} + target_name = read_string(f) + return {'change_type': change_type, 'keyspace': keyspace, target.lower(): target_name} else: return {'change_type': change_type, 'keyspace': keyspace} else: diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index e5b2eab3ce..367b9ee7fe 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -928,3 +928,83 @@ def test_keyspace_alter(self): new_keyspace_meta = self.cluster.metadata.keyspaces[name] self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) self.assertEqual(new_keyspace_meta.durable_writes, False) + +from cassandra.cqltypes import DoubleType +from cassandra.metadata import Function + + +class FunctionMetadata(unittest.TestCase): + + keyspace_name = "functionmetadatatest" + + @property + def function_name(self): + return self._testMethodName.lower() + + @classmethod + def setup_class(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Function metadata requires native protocol version 4+") + + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.session = cls.cluster.connect() + cls.session.execute("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}" % cls.keyspace_name) + cls.keyspace_function_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].functions + + @classmethod + def teardown_class(cls): + cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) + cls.cluster.shutdown() + + def make_function_kwargs(self, deterministic=True, called_on_null=True): + return {'keyspace': self.keyspace_name, + 'name': self.function_name, + 'signature': ['double', 'int'], + 'argument_names': ['d', 'i'], + 'return_type': DoubleType, + 'language': 'java', + 'body': 'return new Double(0.0);', + 'is_deterministic': deterministic, + 'called_on_null_input': called_on_null} + + def test_create_drop_function(self): + self.assertNotIn(self.function_name, self.keyspace_function_meta) + + expected_meta = Function(**self.make_function_kwargs()) + self.session.execute(expected_meta.as_cql_query()) + self.assertIn(self.function_name, self.keyspace_function_meta) + + generated_meta = self.keyspace_function_meta[self.function_name] + self.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) + + self.session.execute("DROP FUNCTION %s.%s" % (self.keyspace_name, self.function_name)) + self.assertNotIn(self.function_name, self.keyspace_function_meta) + + # TODO: this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen + @unittest.expectedFailure + def test_functions_after_udt(self): + self.assertNotIn(self.function_name, self.keyspace_function_meta) + + udt_name = 'udtx' + self.session.execute("CREATE TYPE %s.%s (x int)" % (self.keyspace_name, udt_name)) + + # make a function that takes a udt type + kwargs = self.make_function_kwargs() + kwargs['signature'][0] = "frozen<%s>" % udt_name + + expected_meta = Function(**kwargs) + self.session.execute(expected_meta.as_cql_query()) + self.assertIn(self.function_name, self.keyspace_function_meta) + + generated_meta = self.keyspace_function_meta[self.function_name] + self.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) + + # udts must come before functions in keyspace dump + keyspace_cql = self.cluster.metadata.keyspaces[self.keyspace_name].export_as_string() + type_idx = keyspace_cql.rfind("CREATE TYPE") + func_idx = keyspace_cql.find("CREATE FUCNTION") + self.assertNotIn(-1, (type_idx, func_idx)) + self.assertGreater(func_idx, type_idx) + + self.session.execute("DROP FUNCTION %s.%s" % (self.keyspace_name, self.function_name)) + self.assertNotIn(self.function_name, self.keyspace_function_meta) From 1e420158a9dd6124f449205a3fe9424b86076288 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Apr 2015 16:19:24 -0500 Subject: [PATCH 0017/2431] improve and expond function metadata tests --- tests/integration/standard/test_metadata.py | 100 ++++++++++++++------ 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 367b9ee7fe..b8df0738e7 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -956,6 +956,27 @@ def teardown_class(cls): cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) cls.cluster.shutdown() + class VerifiedFunction(object): + def __init__(self, test_case, **function_kwargs): + self.test_case = test_case + self.function_kwargs = function_kwargs + + def __enter__(self): + tc = self.test_case + expected_meta = Function(**self.function_kwargs) + tc.assertNotIn(expected_meta.name, tc.keyspace_function_meta) + tc.session.execute(expected_meta.as_cql_query()) + tc.assertIn(expected_meta.name, tc.keyspace_function_meta) + + generated_meta = tc.keyspace_function_meta[expected_meta.name] + self.test_case.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) + + def __exit__(self, exc_type, exc_val, exc_tb): + tc = self.test_case + function_name = self.function_kwargs['name'] + tc.session.execute("DROP FUNCTION %s.%s" % (tc.keyspace_name, function_name)) + tc.assertNotIn(function_name, tc.keyspace_function_meta) + def make_function_kwargs(self, deterministic=True, called_on_null=True): return {'keyspace': self.keyspace_name, 'name': self.function_name, @@ -968,43 +989,62 @@ def make_function_kwargs(self, deterministic=True, called_on_null=True): 'called_on_null_input': called_on_null} def test_create_drop_function(self): - self.assertNotIn(self.function_name, self.keyspace_function_meta) - - expected_meta = Function(**self.make_function_kwargs()) - self.session.execute(expected_meta.as_cql_query()) - self.assertIn(self.function_name, self.keyspace_function_meta) - - generated_meta = self.keyspace_function_meta[self.function_name] - self.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) - - self.session.execute("DROP FUNCTION %s.%s" % (self.keyspace_name, self.function_name)) - self.assertNotIn(self.function_name, self.keyspace_function_meta) + with self.VerifiedFunction(self, **self.make_function_kwargs()): + pass - # TODO: this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen - @unittest.expectedFailure def test_functions_after_udt(self): self.assertNotIn(self.function_name, self.keyspace_function_meta) udt_name = 'udtx' self.session.execute("CREATE TYPE %s.%s (x int)" % (self.keyspace_name, udt_name)) - # make a function that takes a udt type + # Ideally we would make a function that takes a udt type, but + # this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen + # Maybe update this after release + #kwargs = self.make_function_kwargs() + #kwargs['signature'][0] = "frozen<%s>" % udt_name + + #expected_meta = Function(**kwargs) + #with self.VerifiedFunction(self, **kwargs): + with self.VerifiedFunction(self, **self.make_function_kwargs()): + # udts must come before functions in keyspace dump + keyspace_cql = self.cluster.metadata.keyspaces[self.keyspace_name].export_as_string() + type_idx = keyspace_cql.rfind("CREATE TYPE") + func_idx = keyspace_cql.find("CREATE FUNCTION") + self.assertNotIn(-1, (type_idx, func_idx), "TYPE or FUNCTION not found in keyspace_cql: " + keyspace_cql) + self.assertGreater(func_idx, type_idx) + + def test_functions_follow_keyspace_alter(self): + with self.VerifiedFunction(self, **self.make_function_kwargs()): + original_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.session.execute('ALTER KEYSPACE %s WITH durable_writes = false' % self.keyspace_name) + try: + new_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) + self.assertIs(original_keyspace_meta.functions, new_keyspace_meta.functions) + finally: + self.session.execute('ALTER KEYSPACE %s WITH durable_writes = true' % self.keyspace_name) + + def test_function_cql_determinism(self): kwargs = self.make_function_kwargs() - kwargs['signature'][0] = "frozen<%s>" % udt_name - - expected_meta = Function(**kwargs) - self.session.execute(expected_meta.as_cql_query()) - self.assertIn(self.function_name, self.keyspace_function_meta) - - generated_meta = self.keyspace_function_meta[self.function_name] - self.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) + kwargs['is_deterministic'] = True + with self.VerifiedFunction(self, **kwargs): + fn_meta = self.keyspace_function_meta[self.function_name] + self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*") - # udts must come before functions in keyspace dump - keyspace_cql = self.cluster.metadata.keyspaces[self.keyspace_name].export_as_string() - type_idx = keyspace_cql.rfind("CREATE TYPE") - func_idx = keyspace_cql.find("CREATE FUCNTION") - self.assertNotIn(-1, (type_idx, func_idx)) - self.assertGreater(func_idx, type_idx) + kwargs['is_deterministic'] = False + with self.VerifiedFunction(self, **kwargs): + fn_meta = self.keyspace_function_meta[self.function_name] + self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE NON DETERMINISTIC FUNCTION.*") - self.session.execute("DROP FUNCTION %s.%s" % (self.keyspace_name, self.function_name)) - self.assertNotIn(self.function_name, self.keyspace_function_meta) + def test_function_cql_called_on_null(self): + kwargs = self.make_function_kwargs() + kwargs['called_on_null_input'] = True + with self.VerifiedFunction(self, **kwargs): + fn_meta = self.keyspace_function_meta[self.function_name] + self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) CALLED ON NULL INPUT RETURNS .*") + + kwargs['called_on_null_input'] = False + with self.VerifiedFunction(self, **kwargs): + fn_meta = self.keyspace_function_meta[self.function_name] + self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) NOT CALLED ON NULL INPUT RETURNS .*") From fd9df7bd58f68b6f05dd85d1d0fcad090890af75 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Apr 2015 16:20:59 -0500 Subject: [PATCH 0018/2431] commit to called_on_null_input schema for CASSANDRA-8374 --- cassandra/metadata.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index d6cd8e5840..abb2d60a62 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -239,8 +239,7 @@ def _build_function(self, keyspace, function_row): return Function(function_row['keyspace_name'], function_row['function_name'], function_row['signature'], function_row['argument_names'], return_type, function_row['language'], function_row['body'], - function_row['is_deterministic'], function_row.get('called_on_null_input')) - # called_on_null_input is not yet merged + function_row['is_deterministic'], function_row['called_on_null_input']) def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] @@ -786,7 +785,10 @@ def export_as_string(self): Returns a CQL query string that can be used to recreate the entire keyspace, including user-defined types and tables. """ - return "\n\n".join([self.as_cql_query()] + self.user_type_strings() + [t.export_as_string() for t in self.tables.values()]) + return "\n\n".join([self.as_cql_query()] + + self.user_type_strings() + + [f.as_cql_query(True) for f in self.functions.values()] + + [t.export_as_string() for t in self.tables.values()]) def as_cql_query(self): """ @@ -960,8 +962,10 @@ def as_cql_query(self, formatted=False): typ = self.return_type.cql_parameterized_type() lang = self.language body = protect_value(self.body) + on_null = "CALLED" if self.called_on_null_input else "RETURNS NULL" return "CREATE %(determ)sFUNCTION %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ + "%(on_null)s ON NULL INPUT%(sep)s" \ "RETURNS %(typ)s%(sep)s" \ "LANGUAGE %(lang)s%(sep)s" \ "AS %(body)s;" % locals() From 527c6c05bca436c6f84651a333872e5cf5ea5c8d Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 13 Apr 2015 14:19:37 -0500 Subject: [PATCH 0019/2431] Handle overloaded user functions with differing type signatures --- cassandra/__init__.py | 25 +++++++++++++++++++++++++ cassandra/cluster.py | 9 +++++---- cassandra/metadata.py | 21 +++++++++++++-------- cassandra/protocol.py | 13 ++++++++----- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 296601cbf1..d1ad0afe0d 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -302,3 +302,28 @@ class UnsupportedOperation(Exception): for more details. """ pass + + +class UserFunctionDescriptor(object): + """ + Describes a User function or aggregate by name and argument signature + """ + + name = None + + type_signature = None + """ + Ordered list of CQL argument types + """ + + def __init__(self, name, type_signature): + self.name = name + self.type_signature = type_signature + + @property + def signature(self): + return self.format_signature(self.name, self.type_signature) + + @staticmethod + def format_signature(name, type_signature): + return "%s(%s)" % (name, ','.join(t for t in type_signature)) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 96eb4044c9..e1f5ec6f6c 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2076,10 +2076,11 @@ def _handle_results(success, result): self._cluster.metadata.usertype_changed(keyspace, usertype, types_result) elif function: # user defined function within this keyspace changed - where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s'" % (keyspace, function) + where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s' AND signature = [%s]" \ + % (keyspace, function.name, ','.join("'%s'" % t for t in function.type_signature)) functions_query = QueryMessage(query=self._SELECT_FUNCTIONS + where_clause, consistency_level=cl) functions_result = connection.wait_for_response(functions_query) - log.debug("[control connection] Fetched user function info for %s.%s, rebuilding metadata", keyspace, function) + log.debug("[control connection] Fetched user function info for %s.%s, rebuilding metadata", keyspace, function.signature) functions_result = dict_factory(*functions_result.results) if functions_result.results else {} self._cluster.metadata.function_changed(keyspace, function, functions_result) elif keyspace: @@ -2305,7 +2306,7 @@ def _handle_schema_change(self, event): keyspace = event.get('keyspace') table = event.get('table') usertype = event.get('type') - function = event.get('function', event.get('aggregate')) + function = event.get('function') delay = random() * self._schema_event_refresh_window self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype, function) @@ -2733,7 +2734,7 @@ def _set_result(self, response): response.results['keyspace'], response.results.get('table'), response.results.get('type'), - response.results.get('function', response.results.get('aggregate')), + response.results.get('function'), self.session.cluster.control_connection, self) else: diff --git a/cassandra/metadata.py b/cassandra/metadata.py index abb2d60a62..32edd01f9c 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -30,6 +30,7 @@ except ImportError as e: pass +from cassandra import UserFunctionDescriptor import cassandra.cqltypes as types from cassandra.marshal import varint_unpack from cassandra.pool import Host @@ -139,7 +140,7 @@ def rebuild_schema(self, ks_results, type_results, function_results, for fn_row in fn_rows.get(keyspace_meta.name, []): fn = self._build_function(keyspace_meta.name, fn_row) - keyspace_meta.functions[fn.name] = fn + keyspace_meta.functions[fn.signature] = fn current_keyspaces.add(keyspace_meta.name) old_keyspace_meta = self.keyspaces.get(keyspace_meta.name, None) @@ -184,13 +185,13 @@ def usertype_changed(self, keyspace, name, type_results): # the type was deleted self.keyspaces[keyspace].user_types.pop(name, None) - def function_changed(self, keyspace, name, function_results): + def function_changed(self, keyspace, function, function_results): if function_results: new_function = self._build_function(keyspace, function_results[0]) - self.keyspaces[keyspace].functions[name] = new_function + self.keyspaces[keyspace].functions[function.signature] = new_function else: # the function was deleted - self.keyspaces[keyspace].functions.pop(name, None) + self.keyspaces[keyspace].functions.pop(function.signature, None) def table_changed(self, keyspace, table, cf_results, col_results, triggers_result): try: @@ -898,7 +899,7 @@ class Function(object): The name of this function """ - signature = None + type_signature = None """ An ordered list of the types for each argument to the function """ @@ -935,11 +936,11 @@ class Function(object): (convenience function to avoid handling nulls explicitly if the result will just be null) """ - def __init__(self, keyspace, name, signature, argument_names, + def __init__(self, keyspace, name, type_signature, argument_names, return_type, language, body, is_deterministic, called_on_null_input): self.keyspace = keyspace self.name = name - self.signature = signature + self.type_signature = type_signature self.argument_names = argument_names self.return_type = return_type self.language = language @@ -957,7 +958,7 @@ def as_cql_query(self, formatted=False): keyspace = protect_name(self.keyspace) name = protect_name(self.name) arg_list = ', '.join(["%s %s" % (protect_name(n), t) - for n, t in zip(self.argument_names, self.signature)]) + for n, t in zip(self.argument_names, self.type_signature)]) determ = '' if self.is_deterministic else 'NON DETERMINISTIC ' typ = self.return_type.cql_parameterized_type() lang = self.language @@ -970,6 +971,10 @@ def as_cql_query(self, formatted=False): "LANGUAGE %(lang)s%(sep)s" \ "AS %(body)s;" % locals() + @property + def signature(self): + return UserFunctionDescriptor.format_signature(self.name, self.type_signature) + class TableMetadata(object): """ diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 922925a526..9ec114e740 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -23,7 +23,7 @@ from cassandra import (Unavailable, WriteTimeout, ReadTimeout, AlreadyExists, InvalidRequest, Unauthorized, - UnsupportedOperation) + UnsupportedOperation, UserFunctionDescriptor) from cassandra.marshal import (int32_pack, int32_unpack, uint16_pack, uint16_unpack, int8_pack, int8_unpack, uint64_pack, header_pack, v3_header_pack) @@ -850,15 +850,18 @@ def recv_schema_change(cls, f, protocol_version): if protocol_version >= 3: target = read_string(f) keyspace = read_string(f) + event = {'change_type': change_type, 'keyspace': keyspace} if target != "KEYSPACE": target_name = read_string(f) - return {'change_type': change_type, 'keyspace': keyspace, target.lower(): target_name} - else: - return {'change_type': change_type, 'keyspace': keyspace} + if target in ('FUNCTION', 'AGGREGATE'): + event['function'] = UserFunctionDescriptor(target_name, [read_string(f) for _ in range(read_short(f))]) + else: + event[target.lower()] = target_name else: keyspace = read_string(f) table = read_string(f) - return {'change_type': change_type, 'keyspace': keyspace, 'table': table} + event = {'change_type': change_type, 'keyspace': keyspace, 'table': table} + return event def write_header(f, version, flags, stream_id, opcode, length): From d2a0931c7db658d590030361f1887e61d510d3fb Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 13 Apr 2015 14:22:05 -0500 Subject: [PATCH 0020/2431] Finalize user function meta integration tests - handle signatures - new signature overload test - proper called_on_null regex --- tests/integration/standard/test_metadata.py | 56 ++++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index b8df0738e7..05c8ba5611 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -22,7 +22,7 @@ import six import sys -from cassandra import AlreadyExists +from cassandra import AlreadyExists, UserFunctionDescriptor from cassandra.cluster import Cluster from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, @@ -959,28 +959,33 @@ def teardown_class(cls): class VerifiedFunction(object): def __init__(self, test_case, **function_kwargs): self.test_case = test_case - self.function_kwargs = function_kwargs + self.function_kwargs = dict(function_kwargs) def __enter__(self): tc = self.test_case expected_meta = Function(**self.function_kwargs) - tc.assertNotIn(expected_meta.name, tc.keyspace_function_meta) + tc.assertNotIn(expected_meta.signature, tc.keyspace_function_meta) tc.session.execute(expected_meta.as_cql_query()) - tc.assertIn(expected_meta.name, tc.keyspace_function_meta) + tc.assertIn(expected_meta.signature, tc.keyspace_function_meta) - generated_meta = tc.keyspace_function_meta[expected_meta.name] + generated_meta = tc.keyspace_function_meta[expected_meta.signature] self.test_case.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) + return self def __exit__(self, exc_type, exc_val, exc_tb): tc = self.test_case - function_name = self.function_kwargs['name'] - tc.session.execute("DROP FUNCTION %s.%s" % (tc.keyspace_name, function_name)) - tc.assertNotIn(function_name, tc.keyspace_function_meta) + tc.session.execute("DROP FUNCTION %s.%s" % (tc.keyspace_name, self.signature)) + tc.assertNotIn(self.signature, tc.keyspace_function_meta) + + @property + def signature(self): + return UserFunctionDescriptor.format_signature(self.function_kwargs['name'], + self.function_kwargs['type_signature']) def make_function_kwargs(self, deterministic=True, called_on_null=True): return {'keyspace': self.keyspace_name, 'name': self.function_name, - 'signature': ['double', 'int'], + 'type_signature': ['double', 'int'], 'argument_names': ['d', 'i'], 'return_type': DoubleType, 'language': 'java', @@ -1002,7 +1007,7 @@ def test_functions_after_udt(self): # this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen # Maybe update this after release #kwargs = self.make_function_kwargs() - #kwargs['signature'][0] = "frozen<%s>" % udt_name + #kwargs['type_signature'][0] = "frozen<%s>" % udt_name #expected_meta = Function(**kwargs) #with self.VerifiedFunction(self, **kwargs): @@ -1014,6 +1019,19 @@ def test_functions_after_udt(self): self.assertNotIn(-1, (type_idx, func_idx), "TYPE or FUNCTION not found in keyspace_cql: " + keyspace_cql) self.assertGreater(func_idx, type_idx) + def test_function_same_name_diff_types(self): + kwargs = self.make_function_kwargs() + with self.VerifiedFunction(self, **kwargs): + # another function: same name, different type sig. + self.assertGreater(len(kwargs['type_signature']), 1) + self.assertGreater(len(kwargs['argument_names']), 1) + kwargs['type_signature'] = kwargs['type_signature'][:1] + kwargs['argument_names'] = kwargs['argument_names'][:1] + with self.VerifiedFunction(self, **kwargs): + functions = [f for f in self.keyspace_function_meta.values() if f.name == self.function_name] + self.assertEqual(len(functions), 2) + self.assertNotEqual(functions[0].type_signature, functions[1].type_signature) + def test_functions_follow_keyspace_alter(self): with self.VerifiedFunction(self, **self.make_function_kwargs()): original_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] @@ -1028,23 +1046,23 @@ def test_functions_follow_keyspace_alter(self): def test_function_cql_determinism(self): kwargs = self.make_function_kwargs() kwargs['is_deterministic'] = True - with self.VerifiedFunction(self, **kwargs): - fn_meta = self.keyspace_function_meta[self.function_name] + with self.VerifiedFunction(self, **kwargs) as vf: + fn_meta = self.keyspace_function_meta[vf.signature] self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*") kwargs['is_deterministic'] = False - with self.VerifiedFunction(self, **kwargs): - fn_meta = self.keyspace_function_meta[self.function_name] + with self.VerifiedFunction(self, **kwargs) as vf: + fn_meta = self.keyspace_function_meta[vf.signature] self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE NON DETERMINISTIC FUNCTION.*") def test_function_cql_called_on_null(self): kwargs = self.make_function_kwargs() kwargs['called_on_null_input'] = True - with self.VerifiedFunction(self, **kwargs): - fn_meta = self.keyspace_function_meta[self.function_name] + with self.VerifiedFunction(self, **kwargs) as vf: + fn_meta = self.keyspace_function_meta[vf.signature] self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) CALLED ON NULL INPUT RETURNS .*") kwargs['called_on_null_input'] = False - with self.VerifiedFunction(self, **kwargs): - fn_meta = self.keyspace_function_meta[self.function_name] - self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) NOT CALLED ON NULL INPUT RETURNS .*") + with self.VerifiedFunction(self, **kwargs) as vf: + fn_meta = self.keyspace_function_meta[vf.signature] + self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) RETURNS NULL ON NULL INPUT RETURNS .*") From 446a10687d0ef5b2b30482e3c37d57cf76b3a752 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 13 Apr 2015 15:12:48 -0500 Subject: [PATCH 0021/2431] User function API docs --- cassandra/__init__.py | 58 ++++++++++++++++++++++++------------------ cassandra/cluster.py | 31 +++++++++++++++++----- docs/api/cassandra.rst | 3 +++ 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index d1ad0afe0d..2d41a8eaed 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -126,6 +126,39 @@ def consistency_value_to_name(value): return ConsistencyLevel.value_to_name[value] if value is not None else "Not Set" +class UserFunctionDescriptor(object): + """ + Describes a User function or aggregate by name and argument signature + """ + + name = None + """ + name of the function + """ + + type_signature = None + """ + Ordered list of CQL argument type name comprising the type signature + """ + + def __init__(self, name, type_signature): + self.name = name + self.type_signature = type_signature + + @property + def signature(self): + """ + function signatue string in the form 'name([type0[,type1[...]]])' + + can be used to uniquely identify overloaded function names within a keyspace + """ + return self.format_signature(self.name, self.type_signature) + + @staticmethod + def format_signature(name, type_signature): + return "%s(%s)" % (name, ','.join(t for t in type_signature)) + + class Unavailable(Exception): """ There were not enough live replicas to satisfy the requested consistency @@ -302,28 +335,3 @@ class UnsupportedOperation(Exception): for more details. """ pass - - -class UserFunctionDescriptor(object): - """ - Describes a User function or aggregate by name and argument signature - """ - - name = None - - type_signature = None - """ - Ordered list of CQL argument types - """ - - def __init__(self, name, type_signature): - self.name = name - self.type_signature = type_signature - - @property - def signature(self): - return self.format_signature(self.name, self.type_signature) - - @staticmethod - def format_signature(name, type_signature): - return "%s(%s)" % (name, ','.join(t for t in type_signature)) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index e1f5ec6f6c..b3172e788e 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1116,9 +1116,27 @@ def _ensure_core_connections(self): for pool in session._pools.values(): pool.ensure_core_connections() + def _validate_refresh_schema(self, keyspace, table, usertype, function): + if any((table, usertype, function)): + if not keyspace: + raise ValueError("keyspace is required to refresh specific sub-entity {table, usertype, function}") + if sum(1 for e in (table, usertype, function) if e) > 1: + raise ValueError("{table, usertype, function} are mutually exclusive") + def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, max_schema_agreement_wait=None): """ - Synchronously refresh the schema metadata. + Synchronously refresh all schema metadata. + + {keyspace, table, usertype} are string names of the respective entities. + ``function`` is a :class:`cassandra.UserFunctionDescriptor`. + + If none of ``{keyspace, table, usertype, function}`` are specified, the entire schema is refreshed. + + If any of ``{keyspace, table, usertype, function}`` are specified, ``keyspace`` is required. + + If only ``keyspace`` is specified, just the top-level keyspace metadata is refreshed (e.g. replication). + + The remaining arguments ``{table, usertype, function}`` are mutually exclusive -- only one may be specified. By default, the timeout for this operation is governed by :attr:`~.Cluster.max_schema_agreement_wait` and :attr:`~.Cluster.control_connection_timeout`. @@ -1129,17 +1147,18 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None An Exception is raised if schema refresh fails for any reason. """ + self._validate_refresh_schema(keyspace, table, usertype, function) if not self.control_connection.refresh_schema(keyspace, table, usertype, function, max_schema_agreement_wait): raise Exception("Schema was not refreshed. See log for details.") - def submit_schema_refresh(self, keyspace=None, table=None, usertype=None): + def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, function=None): """ Schedule a refresh of the internal representation of the current - schema for this cluster. If `keyspace` is specified, only that - keyspace will be refreshed, and likewise for `table`. + schema for this cluster. See :meth:`~.refresh_schema` for description of parameters. """ + self._validate_refresh_schema(keyspace, table, usertype, function) return self.executor.submit( - self.control_connection.refresh_schema, keyspace, table, usertype) + self.control_connection.refresh_schema, keyspace, table, usertype, function) def refresh_nodes(self): """ @@ -2030,7 +2049,7 @@ def _refresh_schema(self, connection, keyspace=None, table=None, usertype=None, if self._cluster.is_shutdown: return False - assert table is None or usertype is None + assert sum(1 for arg in (table, usertype, function) if arg) <= 1 agreed = self.wait_for_schema_agreement(connection, preloaded_results=preloaded_results, diff --git a/docs/api/cassandra.rst b/docs/api/cassandra.rst index 90d23d108e..91500d0f8a 100644 --- a/docs/api/cassandra.rst +++ b/docs/api/cassandra.rst @@ -14,6 +14,9 @@ .. autoclass:: ConsistencyLevel :members: +.. autoclass:: UserFunctionDescriptor + :members: + .. autoexception:: Unavailable() :members: From bd51fb978663cee3ec77295dad9ca3b233a69564 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 14 Apr 2015 16:51:38 -0500 Subject: [PATCH 0022/2431] Add meta handling for AGGREGATE --- cassandra/__init__.py | 47 ++++++++++----- cassandra/cluster.py | 80 +++++++++++++++++-------- cassandra/metadata.py | 136 ++++++++++++++++++++++++++++++++++++++++-- cassandra/protocol.py | 7 ++- 4 files changed, 223 insertions(+), 47 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 2d41a8eaed..10fd6158a6 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -126,20 +126,7 @@ def consistency_value_to_name(value): return ConsistencyLevel.value_to_name[value] if value is not None else "Not Set" -class UserFunctionDescriptor(object): - """ - Describes a User function or aggregate by name and argument signature - """ - - name = None - """ - name of the function - """ - - type_signature = None - """ - Ordered list of CQL argument type name comprising the type signature - """ +class SignatureDescriptor(object): def __init__(self, name, type_signature): self.name = name @@ -159,6 +146,38 @@ def format_signature(name, type_signature): return "%s(%s)" % (name, ','.join(t for t in type_signature)) +class UserFunctionDescriptor(SignatureDescriptor): + """ + Describes a User function by name and argument signature + """ + + name = None + """ + name of the function + """ + + type_signature = None + """ + Ordered list of CQL argument type name comprising the type signature + """ + + +class UserAggregateDescriptor(SignatureDescriptor): + """ + Describes a User aggregate function by name and argument signature + """ + + name = None + """ + name of the aggregate + """ + + type_signature = None + """ + Ordered list of CQL argument type name comprising the type signature + """ + + class Unavailable(Exception): """ There were not enough live replicas to satisfy the requested consistency diff --git a/cassandra/cluster.py b/cassandra/cluster.py index b3172e788e..8832bb8f3a 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1116,27 +1116,29 @@ def _ensure_core_connections(self): for pool in session._pools.values(): pool.ensure_core_connections() - def _validate_refresh_schema(self, keyspace, table, usertype, function): - if any((table, usertype, function)): + def _validate_refresh_schema(self, keyspace, table, usertype, function, aggregate): + if any((table, usertype, function, aggregate)): if not keyspace: - raise ValueError("keyspace is required to refresh specific sub-entity {table, usertype, function}") + raise ValueError("keyspace is required to refresh specific sub-entity {table, usertype, function, aggregate}") if sum(1 for e in (table, usertype, function) if e) > 1: - raise ValueError("{table, usertype, function} are mutually exclusive") + raise ValueError("{table, usertype, function, aggregate} are mutually exclusive") - def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, max_schema_agreement_wait=None): + def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, aggregate=None, max_schema_agreement_wait=None): """ - Synchronously refresh all schema metadata. + Synchronously refresh schema metadata. {keyspace, table, usertype} are string names of the respective entities. ``function`` is a :class:`cassandra.UserFunctionDescriptor`. + ``aggregate`` is a :class:`cassandra.UserAggregateDescriptor`. - If none of ``{keyspace, table, usertype, function}`` are specified, the entire schema is refreshed. + If none of ``{keyspace, table, usertype, function, aggregate}`` are specified, the entire schema is refreshed. - If any of ``{keyspace, table, usertype, function}`` are specified, ``keyspace`` is required. + If any of ``{keyspace, table, usertype, function, aggregate}`` are specified, ``keyspace`` is required. If only ``keyspace`` is specified, just the top-level keyspace metadata is refreshed (e.g. replication). - The remaining arguments ``{table, usertype, function}`` are mutually exclusive -- only one may be specified. + The remaining arguments ``{table, usertype, function, aggregate}`` + are mutually exclusive -- only one may be specified. By default, the timeout for this operation is governed by :attr:`~.Cluster.max_schema_agreement_wait` and :attr:`~.Cluster.control_connection_timeout`. @@ -1147,8 +1149,9 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None An Exception is raised if schema refresh fails for any reason. """ - self._validate_refresh_schema(keyspace, table, usertype, function) - if not self.control_connection.refresh_schema(keyspace, table, usertype, function, max_schema_agreement_wait): + self._validate_refresh_schema(keyspace, table, usertype, function, aggregate) + if not self.control_connection.refresh_schema(keyspace, table, usertype, function, + aggregate, max_schema_agreement_wait): raise Exception("Schema was not refreshed. See log for details.") def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, function=None): @@ -1156,9 +1159,9 @@ def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, functi Schedule a refresh of the internal representation of the current schema for this cluster. See :meth:`~.refresh_schema` for description of parameters. """ - self._validate_refresh_schema(keyspace, table, usertype, function) + self._validate_refresh_schema(keyspace, table, usertype, function, aggregate) return self.executor.submit( - self.control_connection.refresh_schema, keyspace, table, usertype, function) + self.control_connection.refresh_schema, keyspace, table, usertype, function, aggregate) def refresh_nodes(self): """ @@ -1840,6 +1843,7 @@ class ControlConnection(object): _SELECT_COLUMNS = "SELECT * FROM system.schema_columns" _SELECT_USERTYPES = "SELECT * FROM system.schema_usertypes" _SELECT_FUNCTIONS = "SELECT * FROM system.schema_functions" + _SELECT_AGGREGATES = "SELECT * FROM system.schema_aggregates" _SELECT_TRIGGERS = "SELECT * FROM system.schema_triggers" _SELECT_PEERS = "SELECT peer, data_center, rack, tokens, rpc_address, schema_version FROM system.peers" @@ -2028,7 +2032,7 @@ def shutdown(self): del self._connection def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, - schema_agreement_wait=None): + aggregate=None, schema_agreement_wait=None): if not self._meta_refresh_enabled: log.debug("[control connection] Skipping schema refresh because meta refresh is disabled") return False @@ -2036,7 +2040,7 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None try: if self._connection: return self._refresh_schema(self._connection, keyspace, table, usertype, function, - schema_agreement_wait=schema_agreement_wait) + aggregate, schema_agreement_wait=schema_agreement_wait) except ReferenceError: pass # our weak reference to the Cluster is no good except Exception: @@ -2045,11 +2049,11 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None return False def _refresh_schema(self, connection, keyspace=None, table=None, usertype=None, function=None, - preloaded_results=None, schema_agreement_wait=None): + aggregate=None, preloaded_results=None, schema_agreement_wait=None): if self._cluster.is_shutdown: return False - assert sum(1 for arg in (table, usertype, function) if arg) <= 1 + assert sum(1 for arg in (table, usertype, function, aggregate) if arg) <= 1 agreed = self.wait_for_schema_agreement(connection, preloaded_results=preloaded_results, @@ -2102,6 +2106,15 @@ def _handle_results(success, result): log.debug("[control connection] Fetched user function info for %s.%s, rebuilding metadata", keyspace, function.signature) functions_result = dict_factory(*functions_result.results) if functions_result.results else {} self._cluster.metadata.function_changed(keyspace, function, functions_result) + elif aggregate: + # user defined aggregate within this keyspace changed + where_clause = " WHERE keyspace_name = '%s' AND aggregate_name = '%s' AND signature = [%s]" \ + % (keyspace, aggregate.name, ','.join("'%s'" % t for t in aggregate.type_signature)) + aggregates_query = QueryMessage(query=self._SELECT_AGGREGATES + where_clause, consistency_level=cl) + aggregates_result = connection.wait_for_response(aggregates_query) + log.debug("[control connection] Fetched user aggregate info for %s.%s, rebuilding metadata", keyspace, aggregate.signature) + aggregates_result = dict_factory(*aggregates_result.results) if aggregates_result.results else {} + self._cluster.metadata.aggregate_changed(keyspace, aggregate, aggregates_result) elif keyspace: # only the keyspace itself changed (such as replication settings) where_clause = " WHERE keyspace_name = '%s'" % (keyspace,) @@ -2118,6 +2131,7 @@ def _handle_results(success, result): QueryMessage(query=self._SELECT_COLUMNS, consistency_level=cl), QueryMessage(query=self._SELECT_USERTYPES, consistency_level=cl), QueryMessage(query=self._SELECT_FUNCTIONS, consistency_level=cl), + QueryMessage(query=self._SELECT_AGGREGATES, consistency_level=cl), QueryMessage(query=self._SELECT_TRIGGERS, consistency_level=cl) ] @@ -2125,6 +2139,7 @@ def _handle_results(success, result): (ks_success, ks_result), (cf_success, cf_result), \ (col_success, col_result), (types_success, types_result), \ (functions_success, functions_result), \ + (aggregates_success, aggregates_result), \ (trigger_success, triggers_result) = responses if ks_success: @@ -2176,8 +2191,19 @@ def _handle_results(success, result): else: raise functions_result + # aggregates were introduced in Cassandra 3.0 + if aggregates_success: + aggregates_result = dict_factory(*aggregates_result.results) if aggregates_result.results else {} + else: + if isinstance(aggregates_result, InvalidRequest): + log.debug("[control connection] user aggregates table not found") + aggregates_result = {} + else: + raise aggregates_result + log.debug("[control connection] Fetched schema, rebuilding metadata") - self._cluster.metadata.rebuild_schema(ks_result, types_result, functions_result, cf_result, col_result, triggers_result) + self._cluster.metadata.rebuild_schema(ks_result, types_result, functions_result, + aggregates_result, cf_result, col_result, triggers_result) return True def refresh_node_list_and_token_map(self, force_token_rebuild=False): @@ -2321,13 +2347,13 @@ def _handle_status_change(self, event): def _handle_schema_change(self, event): if self._schema_event_refresh_window < 0: return - keyspace = event.get('keyspace') table = event.get('table') usertype = event.get('type') function = event.get('function') + aggregate = event.get('aggregate') delay = random() * self._schema_event_refresh_window - self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype, function) + self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype, function, aggregate) def wait_for_schema_agreement(self, connection=None, preloaded_results=None, wait_time=None): @@ -2556,19 +2582,20 @@ def _log_if_failed(self, future): exc_info=exc) -def refresh_schema_and_set_result(keyspace, table, usertype, function, control_conn, response_future): +def refresh_schema_and_set_result(keyspace, table, usertype, function, aggregate, control_conn, response_future): try: if control_conn._meta_refresh_enabled: - log.debug("Refreshing schema in response to schema change. Keyspace: %s; Table: %s, Type: %s, Function: %s", - keyspace, table, usertype, function) - control_conn._refresh_schema(response_future._connection, keyspace, table, usertype, function) + log.debug("Refreshing schema in response to schema change. " + "Keyspace: %s; Table: %s, Type: %s, Function: %s, Aggregate: %s", + keyspace, table, usertype, function, aggregate) + control_conn._refresh_schema(response_future._connection, keyspace, table, usertype, function, aggregate) else: log.debug("Skipping schema refresh in response to schema change because meta refresh is disabled; " - "Keyspace: %s; Table: %s, Type: %s, Function: %s", keyspace, table, usertype, function) + "Keyspace: %s; Table: %s, Type: %s, Function: %s", keyspace, table, usertype, function, aggregate) except Exception: log.exception("Exception refreshing schema in response to schema change:") response_future.session.submit( - control_conn.refresh_schema, keyspace, table, usertype, function) + control_conn.refresh_schema, keyspace, table, usertype, function, aggregate) finally: response_future._set_final_result(None) @@ -2754,6 +2781,7 @@ def _set_result(self, response): response.results.get('table'), response.results.get('type'), response.results.get('function'), + response.results.get('aggregate'), self.session.cluster.control_connection, self) else: diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 32edd01f9c..2b92e51408 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -30,8 +30,9 @@ except ImportError as e: pass -from cassandra import UserFunctionDescriptor +from cassandra import SignatureDescriptor import cassandra.cqltypes as types +from cassandra.encoder import Encoder from cassandra.marshal import varint_unpack from cassandra.pool import Host from cassandra.util import OrderedDict @@ -91,7 +92,7 @@ def export_schema_as_string(self): return "\n".join(ks.export_as_string() for ks in self.keyspaces.values()) def rebuild_schema(self, ks_results, type_results, function_results, - cf_results, col_results, triggers_result): + aggregate_results, cf_results, col_results, triggers_result): """ Rebuild the view of the current schema from a fresh set of rows from the system schema tables. @@ -102,6 +103,7 @@ def rebuild_schema(self, ks_results, type_results, function_results, col_def_rows = defaultdict(lambda: defaultdict(list)) usertype_rows = defaultdict(list) fn_rows = defaultdict(list) + agg_rows = defaultdict(list) trigger_rows = defaultdict(lambda: defaultdict(list)) for row in cf_results: @@ -118,6 +120,9 @@ def rebuild_schema(self, ks_results, type_results, function_results, for row in function_results: fn_rows[row["keyspace_name"]].append(row) + for row in aggregate_results: + agg_rows[row["keyspace_name"]].append(row) + for row in triggers_result: ksname = row["keyspace_name"] cfname = row["columnfamily_name"] @@ -142,6 +147,10 @@ def rebuild_schema(self, ks_results, type_results, function_results, fn = self._build_function(keyspace_meta.name, fn_row) keyspace_meta.functions[fn.signature] = fn + for agg_row in agg_rows.get(keyspace_meta.name, []): + agg = self._build_aggregate(keyspace_meta.name, agg_row) + keyspace_meta.aggregates[agg.signature] = agg + current_keyspaces.add(keyspace_meta.name) old_keyspace_meta = self.keyspaces.get(keyspace_meta.name, None) self.keyspaces[keyspace_meta.name] = keyspace_meta @@ -172,6 +181,7 @@ def keyspace_changed(self, keyspace, ks_results): keyspace_meta.tables = old_keyspace_meta.tables keyspace_meta.user_types = old_keyspace_meta.user_types keyspace_meta.functions = old_keyspace_meta.functions + keyspace_meta.aggregates = old_keyspace_meta.aggregates if (keyspace_meta.replication_strategy != old_keyspace_meta.replication_strategy): self._keyspace_updated(keyspace) else: @@ -193,6 +203,14 @@ def function_changed(self, keyspace, function, function_results): # the function was deleted self.keyspaces[keyspace].functions.pop(function.signature, None) + def aggregate_changed(self, keyspace, aggregate, aggregate_results): + if aggregate_results: + new_aggregate = self._build_aggregate(keyspace, aggregate_results[0]) + self.keyspaces[keyspace].aggregates[aggregate.signature] = new_aggregate + else: + # the aggregate was deleted + self.keyspaces[keyspace].aggregates.pop(aggregate.signature, None) + def table_changed(self, keyspace, table, cf_results, col_results, triggers_result): try: keyspace_meta = self.keyspaces[keyspace] @@ -242,6 +260,16 @@ def _build_function(self, keyspace, function_row): return_type, function_row['language'], function_row['body'], function_row['is_deterministic'], function_row['called_on_null_input']) + def _build_aggregate(self, keyspace, aggregate_row): + state_type = types.lookup_casstype(aggregate_row['state_type']) + initial_condition = aggregate_row['initcond'] + if initial_condition is not None: + initial_condition = state_type.deserialize(initial_condition, 3) + return_type = types.lookup_casstype(aggregate_row['return_type']) + return Aggregate(aggregate_row['keyspace_name'], aggregate_row['aggregate_name'], + aggregate_row['signature'], aggregate_row['final_func'], initial_condition, + return_type, aggregate_row['state_func'], state_type) + def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) @@ -762,14 +790,21 @@ class KeyspaceMetadata(object): user_types = None """ - A map from user-defined type names to instances of :class:`~cassandra.metadata..UserType`. + A map from user-defined type names to instances of :class:`~cassandra.metadata.UserType`. .. versionadded:: 2.1.0 """ functions = None """ - A map from user-defined function names to instances of :class:`~cassandra.metadata..Function`. + A map from user-defined function signatures to instances of :class:`~cassandra.metadata.Function`. + + .. versionadded:: 3.0.0 + """ + + aggregates = None + """ + A map from user-defined aggregate signatures to instances of :class:`~cassandra.metadata.Aggregate`. .. versionadded:: 3.0.0 """ @@ -780,6 +815,7 @@ def __init__(self, name, durable_writes, strategy_class, strategy_options): self.tables = {} self.user_types = {} self.functions = {} + self.aggregates = {} def export_as_string(self): """ @@ -789,6 +825,7 @@ def export_as_string(self): return "\n\n".join([self.as_cql_query()] + self.user_type_strings() + [f.as_cql_query(True) for f in self.functions.values()] + + [a.as_cql_query(True) for a in self.aggregates.values()] + [t.export_as_string() for t in self.tables.values()]) def as_cql_query(self): @@ -880,6 +917,95 @@ def as_cql_query(self, formatted=False): return ret +class Aggregate(object): + """ + A user defined aggregate function, as created by ``CREATE AGGREGATE`` statements. + + Aggregate functions were introduced in Cassandra 3.0 + + .. versionadded:: 3.0.0 + """ + + keyspace = None + """ + The string name of the keyspace in which this aggregate is defined + """ + + name = None + """ + The name of this aggregate + """ + + type_signature = None + """ + An ordered list of the types for each argument to the aggregate + """ + + final_func = None + """ + Name of a final function + """ + + initial_condition = None + """ + Initial condition of the aggregate + """ + + return_type = None + """ + Return type of the aggregate + """ + + state_func = None + """ + Name of a state function + """ + + state_type = None + """ + Flag indicating whether this function is deterministic + (required for functional indexes) + """ + + def __init__(self, keyspace, name, type_signature, final_func, + initial_condition, return_type, state_func, state_type): + self.keyspace = keyspace + self.name = name + self.type_signature = type_signature + self.final_func = final_func + self.initial_condition = initial_condition + self.return_type = return_type + self.state_func = state_func + self.state_type = state_type + + def as_cql_query(self, formatted=False): + """ + Returns a CQL query that can be used to recreate this aggregate. + If `formatted` is set to :const:`True`, extra whitespace will + be added to make the query more readable. + """ + sep = '\n' if formatted else ' ' + keyspace = protect_name(self.keyspace) + name = protect_name(self.name) + arg_list = ', '.join(self.type_signature) + state_func = protect_name(self.state_func) + state_type = self.state_type.cql_parameterized_type() + + ret = "CREATE AGGREGATE %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ + "SFUNC %(state_func)s%(sep)s" \ + "STYPE %(state_type)s" % locals() + + ret += ''.join((sep, 'FINALFUNC ', protect_name(self.final_func))) if self.final_func else '' + ret += ''.join((sep, 'INITCOND ', Encoder().cql_encode_all_types(self.initial_condition)))\ + if self.initial_condition is not None else '' + + return ret + + @property + def signature(self): + return SignatureDescriptor.format_signature(self.name, self.type_signature) + + class Function(object): """ A user defined function, as created by ``CREATE FUNCTION`` statements. @@ -973,7 +1099,7 @@ def as_cql_query(self, formatted=False): @property def signature(self): - return UserFunctionDescriptor.format_signature(self.name, self.type_signature) + return SignatureDescriptor.format_signature(self.name, self.type_signature) class TableMetadata(object): diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 9ec114e740..c6c34615a4 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -23,7 +23,8 @@ from cassandra import (Unavailable, WriteTimeout, ReadTimeout, AlreadyExists, InvalidRequest, Unauthorized, - UnsupportedOperation, UserFunctionDescriptor) + UnsupportedOperation, UserFunctionDescriptor, + UserAggregateDescriptor) from cassandra.marshal import (int32_pack, int32_unpack, uint16_pack, uint16_unpack, int8_pack, int8_unpack, uint64_pack, header_pack, v3_header_pack) @@ -853,8 +854,10 @@ def recv_schema_change(cls, f, protocol_version): event = {'change_type': change_type, 'keyspace': keyspace} if target != "KEYSPACE": target_name = read_string(f) - if target in ('FUNCTION', 'AGGREGATE'): + if target == 'FUNCTION': event['function'] = UserFunctionDescriptor(target_name, [read_string(f) for _ in range(read_short(f))]) + elif target == 'AGGREGATE': + event['aggregate'] = UserAggregateDescriptor(target_name, [read_string(f) for _ in range(read_short(f))]) else: event[target.lower()] = target_name else: From a0482216e8df887d3aa1276703cd3d87239f8148 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 14 Apr 2015 16:52:21 -0500 Subject: [PATCH 0023/2431] Doc update for function and aggregate metadata --- docs/api/cassandra.rst | 5 +++++ docs/api/cassandra/metadata.rst | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/docs/api/cassandra.rst b/docs/api/cassandra.rst index 91500d0f8a..87ef7fe273 100644 --- a/docs/api/cassandra.rst +++ b/docs/api/cassandra.rst @@ -16,6 +16,11 @@ .. autoclass:: UserFunctionDescriptor :members: + :inherited-members: + +.. autoclass:: UserAggregateDescriptor + :members: + :inherited-members: .. autoexception:: Unavailable() :members: diff --git a/docs/api/cassandra/metadata.rst b/docs/api/cassandra/metadata.rst index 2d3b09f766..a89fd03ddb 100644 --- a/docs/api/cassandra/metadata.rst +++ b/docs/api/cassandra/metadata.rst @@ -13,6 +13,15 @@ Schemas .. autoclass:: KeyspaceMetadata () :members: +.. autoclass:: UserType () + :members: + +.. autoclass:: Function () + :members: + +.. autoclass:: Aggregate () + :members: + .. autoclass:: TableMetadata () :members: From cf486b48ce4cae7716d4347d3d171447f40ff6ad Mon Sep 17 00:00:00 2001 From: Anthony Cervantes Date: Tue, 14 Apr 2015 19:08:41 -0500 Subject: [PATCH 0024/2431] fixed typo --- cassandra/cqlengine/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/cqlengine/models.py b/cassandra/cqlengine/models.py index 35d10bb15d..0baa0906a0 100644 --- a/cassandra/cqlengine/models.py +++ b/cassandra/cqlengine/models.py @@ -705,7 +705,7 @@ def update(self, **values): if self._is_polymorphic_base: raise PolymorphicModelException('cannot update polymorphic base model') else: - setattr(self, self._discriminator_column_name, self.__disciminator_value__) + setattr(self, self._discriminator_column_name, self.__discriminator_value__) self.validate() self.__dmlquery__(self.__class__, self, From 37eb549b06a2bbd1f42ca26353a5f83a162c4109 Mon Sep 17 00:00:00 2001 From: Kevin Deldycke Date: Wed, 15 Apr 2015 16:33:06 +0200 Subject: [PATCH 0025/2431] Fix class attribute name. --- docs/cqlengine/upgrade_guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cqlengine/upgrade_guide.rst b/docs/cqlengine/upgrade_guide.rst index 749106de5b..0657a0719c 100644 --- a/docs/cqlengine/upgrade_guide.rst +++ b/docs/cqlengine/upgrade_guide.rst @@ -95,7 +95,7 @@ Model Inheritance The names for class attributes controlling model inheritance are changing. Changes are as follows: - Replace 'polymorphic_key' in the base class Column definition with :attr:`~.discriminator_column` -- Replace the '__polymporphic_key__' class attribute the derived classes with :attr:`~.__discriminator_value__` +- Replace the '__polymorphic_key__' class attribute the derived classes with :attr:`~.__discriminator_value__` The functionality is unchanged -- the intent here is to make the names and language around these attributes more precise. For now, the old names are just deprecated, and the mapper will emit warnings if they are used. The old names From e5943882f2b99b920bbbff2d06920b01e09354bb Mon Sep 17 00:00:00 2001 From: Kevin Deldycke Date: Wed, 15 Apr 2015 16:33:17 +0200 Subject: [PATCH 0026/2431] Remove trailing spaces. --- docs/cqlengine/upgrade_guide.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cqlengine/upgrade_guide.rst b/docs/cqlengine/upgrade_guide.rst index 0657a0719c..3488902790 100644 --- a/docs/cqlengine/upgrade_guide.rst +++ b/docs/cqlengine/upgrade_guide.rst @@ -41,7 +41,7 @@ Package-Level Aliases Legacy cqlengine defined a number of aliases at the package level, which became redundant when the package was integrated for a driver. These have been removed in favor of absolute imports, and referring to cannonical definitions. For example, ``cqlengine.ONE`` was an alias -of ``cassandra.ConsistencyLevel.ONE``. In the integrated package, only the +of ``cassandra.ConsistencyLevel.ONE``. In the integrated package, only the :class:`cassandra.ConsistencyLevel` remains. Additionally, submodule aliases are removed from cqlengine in favor of absolute imports. @@ -70,7 +70,7 @@ IfNotExistsWithCounterColumn cassandra.cqlengine.query UnicodeMixin Consolidation -------------------------- ``class UnicodeMixin`` was defined in several cqlengine modules. This has been consolidated -to a single definition in the cqlengine package init file. This is not technically part of +to a single definition in the cqlengine package init file. This is not technically part of the API, but noted here for completeness. API Deprecations @@ -97,7 +97,7 @@ The names for class attributes controlling model inheritance are changing. Chang - Replace 'polymorphic_key' in the base class Column definition with :attr:`~.discriminator_column` - Replace the '__polymorphic_key__' class attribute the derived classes with :attr:`~.__discriminator_value__` -The functionality is unchanged -- the intent here is to make the names and language around these attributes more precise. +The functionality is unchanged -- the intent here is to make the names and language around these attributes more precise. For now, the old names are just deprecated, and the mapper will emit warnings if they are used. The old names will be removed in a future version. @@ -117,7 +117,7 @@ Before:: class Dog(Pet): __polymorphic_key__ = 'dog' - + After:: class Pet(models.Model): From 3ab95f0adf09ff2ebcbb37271ca28ac418217aa2 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Apr 2015 09:23:45 -0500 Subject: [PATCH 0027/2431] Add OrderedMapSerializedKey to cql encoder mapping This is to correctly encode values that are originally returned from a query. --- cassandra/encoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cassandra/encoder.py b/cassandra/encoder.py index 02eed2aa22..d9c3e85212 100644 --- a/cassandra/encoder.py +++ b/cassandra/encoder.py @@ -28,7 +28,8 @@ from uuid import UUID import six -from cassandra.util import OrderedDict, OrderedMap, sortedset, Time +from cassandra.util import (OrderedDict, OrderedMap, OrderedMapSerializedKey, + sortedset, Time) if six.PY3: long = int @@ -79,6 +80,7 @@ def __init__(self): dict: self.cql_encode_map_collection, OrderedDict: self.cql_encode_map_collection, OrderedMap: self.cql_encode_map_collection, + OrderedMapSerializedKey: self.cql_encode_map_collection, list: self.cql_encode_list_collection, tuple: self.cql_encode_list_collection, set: self.cql_encode_set_collection, From e17c0fe2ee09f300141bb5d7d323e61e88c89c59 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Apr 2015 09:25:55 -0500 Subject: [PATCH 0028/2431] Reorder aggregate args to match grammar order, not table column order. --- cassandra/metadata.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 2b92e51408..b6e0aee7b3 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -267,8 +267,8 @@ def _build_aggregate(self, keyspace, aggregate_row): initial_condition = state_type.deserialize(initial_condition, 3) return_type = types.lookup_casstype(aggregate_row['return_type']) return Aggregate(aggregate_row['keyspace_name'], aggregate_row['aggregate_name'], - aggregate_row['signature'], aggregate_row['final_func'], initial_condition, - return_type, aggregate_row['state_func'], state_type) + aggregate_row['signature'], aggregate_row['state_func'], state_type, + aggregate_row['final_func'], initial_condition, return_type) def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] @@ -967,16 +967,16 @@ class Aggregate(object): (required for functional indexes) """ - def __init__(self, keyspace, name, type_signature, final_func, - initial_condition, return_type, state_func, state_type): + def __init__(self, keyspace, name, type_signature, state_func, + state_type, final_func, initial_condition, return_type): self.keyspace = keyspace self.name = name self.type_signature = type_signature + self.state_func = state_func + self.state_type = state_type self.final_func = final_func self.initial_condition = initial_condition self.return_type = return_type - self.state_func = state_func - self.state_type = state_type def as_cql_query(self, formatted=False): """ @@ -987,11 +987,11 @@ def as_cql_query(self, formatted=False): sep = '\n' if formatted else ' ' keyspace = protect_name(self.keyspace) name = protect_name(self.name) - arg_list = ', '.join(self.type_signature) + type_list = ', '.join(self.type_signature) state_func = protect_name(self.state_func) state_type = self.state_type.cql_parameterized_type() - ret = "CREATE AGGREGATE %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ + ret = "CREATE AGGREGATE %(keyspace)s.%(name)s(%(type_list)s)%(sep)s" \ "SFUNC %(state_func)s%(sep)s" \ "STYPE %(state_type)s" % locals() From 5e5deb17419e4eb3fdefe490c765e536c66f69a8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Apr 2015 09:46:54 -0500 Subject: [PATCH 0029/2431] Aggregate metadata integration tests --- cassandra/cluster.py | 2 +- tests/integration/standard/test_metadata.py | 224 +++++++++++++++++--- 2 files changed, 199 insertions(+), 27 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 8832bb8f3a..e8697120e3 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1154,7 +1154,7 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None aggregate, max_schema_agreement_wait): raise Exception("Schema was not refreshed. See log for details.") - def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, function=None): + def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, function=None, aggregate=None): """ Schedule a refresh of the internal representation of the current schema for this cluster. See :meth:`~.refresh_schema` for description of parameters. diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 05c8ba5611..9c80a2eef0 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -22,11 +22,13 @@ import six import sys -from cassandra import AlreadyExists, UserFunctionDescriptor +from cassandra import AlreadyExists, SignatureDescriptor from cassandra.cluster import Cluster +from cassandra.cqltypes import DoubleType, Int32Type, ListType, UTF8Type, MapType +from cassandra.encoder import Encoder from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, - Token, MD5Token, TokenMap, murmur3) + Token, MD5Token, TokenMap, murmur3, Function, Aggregate) from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host @@ -929,14 +931,11 @@ def test_keyspace_alter(self): self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) self.assertEqual(new_keyspace_meta.durable_writes, False) -from cassandra.cqltypes import DoubleType -from cassandra.metadata import Function - - -class FunctionMetadata(unittest.TestCase): - - keyspace_name = "functionmetadatatest" +class FunctionTest(unittest.TestCase): + """ + Base functionality for Function and Aggregate metadata test classes + """ @property def function_name(self): return self._testMethodName.lower() @@ -947,40 +946,57 @@ def setup_class(cls): raise unittest.SkipTest("Function metadata requires native protocol version 4+") cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.keyspace_name = cls.__name__.lower() cls.session = cls.cluster.connect() - cls.session.execute("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}" % cls.keyspace_name) + cls.session.execute("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}" % cls.keyspace_name) + cls.session.set_keyspace(cls.keyspace_name) cls.keyspace_function_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].functions + cls.keyspace_aggregate_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].aggregates @classmethod def teardown_class(cls): - cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) + cls.session.execute("DROP KEYSPACE IF EXISTS %s" % cls.keyspace_name) cls.cluster.shutdown() - class VerifiedFunction(object): - def __init__(self, test_case, **function_kwargs): + class Verified(object): + + def __init__(self, test_case, meta_class, element_meta, **function_kwargs): self.test_case = test_case self.function_kwargs = dict(function_kwargs) + self.meta_class = meta_class + self.element_meta = element_meta def __enter__(self): tc = self.test_case - expected_meta = Function(**self.function_kwargs) - tc.assertNotIn(expected_meta.signature, tc.keyspace_function_meta) + expected_meta = self.meta_class(**self.function_kwargs) + tc.assertNotIn(expected_meta.signature, self.element_meta) tc.session.execute(expected_meta.as_cql_query()) - tc.assertIn(expected_meta.signature, tc.keyspace_function_meta) + tc.assertIn(expected_meta.signature, self.element_meta) - generated_meta = tc.keyspace_function_meta[expected_meta.signature] + generated_meta = self.element_meta[expected_meta.signature] self.test_case.assertEqual(generated_meta.as_cql_query(), expected_meta.as_cql_query()) return self def __exit__(self, exc_type, exc_val, exc_tb): tc = self.test_case - tc.session.execute("DROP FUNCTION %s.%s" % (tc.keyspace_name, self.signature)) - tc.assertNotIn(self.signature, tc.keyspace_function_meta) + tc.session.execute("DROP %s %s.%s" % (self.meta_class.__name__, tc.keyspace_name, self.signature)) + tc.assertNotIn(self.signature, self.element_meta) @property def signature(self): - return UserFunctionDescriptor.format_signature(self.function_kwargs['name'], - self.function_kwargs['type_signature']) + return SignatureDescriptor.format_signature(self.function_kwargs['name'], + self.function_kwargs['type_signature']) + + class VerifiedFunction(Verified): + def __init__(self, test_case, **kwargs): + super(FunctionTest.VerifiedFunction, self).__init__(test_case, Function, test_case.keyspace_function_meta, **kwargs) + + class VerifiedAggregate(Verified): + def __init__(self, test_case, **kwargs): + super(FunctionTest.VerifiedAggregate, self).__init__(test_case, Aggregate, test_case.keyspace_aggregate_meta, **kwargs) + + +class FunctionMetadata(FunctionTest): def make_function_kwargs(self, deterministic=True, called_on_null=True): return {'keyspace': self.keyspace_name, @@ -993,18 +1009,15 @@ def make_function_kwargs(self, deterministic=True, called_on_null=True): 'is_deterministic': deterministic, 'called_on_null_input': called_on_null} - def test_create_drop_function(self): - with self.VerifiedFunction(self, **self.make_function_kwargs()): - pass - def test_functions_after_udt(self): self.assertNotIn(self.function_name, self.keyspace_function_meta) udt_name = 'udtx' - self.session.execute("CREATE TYPE %s.%s (x int)" % (self.keyspace_name, udt_name)) + self.session.execute("CREATE TYPE %s (x int)" % udt_name) # Ideally we would make a function that takes a udt type, but # this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen + # https://issues.apache.org/jira/browse/CASSANDRA-9186 # Maybe update this after release #kwargs = self.make_function_kwargs() #kwargs['type_signature'][0] = "frozen<%s>" % udt_name @@ -1066,3 +1079,162 @@ def test_function_cql_called_on_null(self): with self.VerifiedFunction(self, **kwargs) as vf: fn_meta = self.keyspace_function_meta[vf.signature] self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) RETURNS NULL ON NULL INPUT RETURNS .*") + + +class AggregateMetadata(FunctionTest): + + @classmethod + def setup_class(cls): + super(AggregateMetadata, cls).setup_class() + + cls.session.execute("""CREATE OR REPLACE FUNCTION sum_int(s int, i int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 's + i';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION sum_int_two(s int, i int, j int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 's + i + j';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION "List_As_String"(l list) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS ''''' + l';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION extend_list(s list, i int) + CALLED ON NULL INPUT + RETURNS list + LANGUAGE java AS 'if (i != null) s.add(i.toString()); return s;';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION update_map(s map, i int) + RETURNS NULL ON NULL INPUT + RETURNS map + LANGUAGE java AS 's.put(new Integer(i), new Integer(i)); return s;';""") + cls.session.execute("""CREATE TABLE IF NOT EXISTS t + (k int PRIMARY KEY, v int)""") + for x in range(4): + cls.session.execute("INSERT INTO t (k,v) VALUES (%s, %s)", (x, x)) + cls.session.execute("INSERT INTO t (k) VALUES (%s)", (4,)) + + def make_aggregate_kwargs(self, state_func, state_type, final_func=None, init_cond=None): + return {'keyspace': self.keyspace_name, + 'name': self.function_name + '_aggregate', + 'type_signature': ['int'], + 'state_func': state_func, + 'state_type': state_type, + 'final_func': final_func, + 'initial_condition': init_cond, + 'return_type': "does not matter for creation"} + + def test_return_type_meta(self): + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=1)) as va: + self.assertIs(self.keyspace_aggregate_meta[va.signature].return_type, Int32Type) + + def test_init_cond(self): + # This is required until the java driver bundled with C* is updated to support v4 + c = Cluster(protocol_version=3) + s = c.connect(self.keyspace_name) + + expected_values = range(4) + + # int32 + for init_cond in (-1, 0, 1): + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=init_cond)) as va: + sum_res = s.execute("SELECT %s(v) AS sum FROM t" % va.function_kwargs['name'])[0].sum + self.assertEqual(sum_res, init_cond + sum(expected_values)) + + # list + for init_cond in ([], ['1', '2']): + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('extend_list', ListType.apply_parameters([UTF8Type]), init_cond=init_cond)) as va: + list_res = s.execute("SELECT %s(v) AS list_res FROM t" % va.function_kwargs['name'])[0].list_res + self.assertListEqual(list_res[:len(init_cond)], init_cond) + self.assertEqual(set(i for i in list_res[len(init_cond):]), + set(str(i) for i in expected_values)) + + # map + expected_map_values = dict((i, i) for i in expected_values) + expected_key_set = set(expected_values) + for init_cond in ({}, {1: 2, 3: 4}, {5: 5}): + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('update_map', MapType.apply_parameters([Int32Type, Int32Type]), init_cond=init_cond)) as va: + map_res = s.execute("SELECT %s(v) AS map_res FROM t" % va.function_kwargs['name'])[0].map_res + self.assertDictContainsSubset(expected_map_values, map_res) + init_not_updated = dict((k, init_cond[k]) for k in set(init_cond) - expected_key_set) + self.assertDictContainsSubset(init_not_updated, map_res) + c.shutdown() + + def test_aggregates_after_functions(self): + # functions must come before functions in keyspace dump + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('extend_list', ListType.apply_parameters([UTF8Type]))): + keyspace_cql = self.cluster.metadata.keyspaces[self.keyspace_name].export_as_string() + func_idx = keyspace_cql.find("CREATE FUNCTION") + aggregate_idx = keyspace_cql.rfind("CREATE AGGREGATE") + self.assertNotIn(-1, (aggregate_idx, func_idx), "AGGREGATE or FUNCTION not found in keyspace_cql: " + keyspace_cql) + self.assertGreater(aggregate_idx, func_idx) + + def test_same_name_diff_types(self): + kwargs = self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=0) + with self.VerifiedAggregate(self, **kwargs): + kwargs['state_func'] = 'sum_int_two' + kwargs['type_signature'] = ['int', 'int'] + with self.VerifiedAggregate(self, **kwargs): + aggregates = [a for a in self.keyspace_aggregate_meta.values() if a.name == kwargs['name']] + self.assertEqual(len(aggregates), 2) + self.assertNotEqual(aggregates[0].type_signature, aggregates[1].type_signature) + + def test_aggregates_follow_keyspace_alter(self): + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=0)): + original_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.session.execute('ALTER KEYSPACE %s WITH durable_writes = false' % self.keyspace_name) + try: + new_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) + self.assertIs(original_keyspace_meta.aggregates, new_keyspace_meta.aggregates) + finally: + self.session.execute('ALTER KEYSPACE %s WITH durable_writes = true' % self.keyspace_name) + + def test_cql_optional_params(self): + kwargs = self.make_aggregate_kwargs('extend_list', ListType.apply_parameters([UTF8Type])) + + # no initial condition, final func + self.assertIsNone(kwargs['initial_condition']) + self.assertIsNone(kwargs['final_func']) + with self.VerifiedAggregate(self, **kwargs) as va: + meta = self.keyspace_aggregate_meta[va.signature] + self.assertIsNone(meta.initial_condition) + self.assertIsNone(meta.final_func) + cql = meta.as_cql_query() + self.assertEqual(cql.find('INITCOND'), -1) + self.assertEqual(cql.find('FINALFUNC'), -1) + + # initial condition, no final func + kwargs['initial_condition'] = ['init', 'cond'] + with self.VerifiedAggregate(self, **kwargs) as va: + meta = self.keyspace_aggregate_meta[va.signature] + self.assertListEqual(meta.initial_condition, kwargs['initial_condition']) + self.assertIsNone(meta.final_func) + cql = meta.as_cql_query() + search_string = "INITCOND %s" % Encoder().cql_encode_all_types(kwargs['initial_condition']) + self.assertGreater(cql.find(search_string), 0, '"%s" search string not found in cql:\n%s' % (search_string, cql)) + self.assertEqual(cql.find('FINALFUNC'), -1) + + # no initial condition, final func + kwargs['initial_condition'] = None + kwargs['final_func'] = 'List_As_String' + with self.VerifiedAggregate(self, **kwargs) as va: + meta = self.keyspace_aggregate_meta[va.signature] + self.assertIsNone(meta.initial_condition) + self.assertEqual(meta.final_func, kwargs['final_func']) + cql = meta.as_cql_query() + self.assertEqual(cql.find('INITCOND'), -1) + search_string = 'FINALFUNC "%s"' % kwargs['final_func'] + self.assertGreater(cql.find(search_string), 0, '"%s" search string not found in cql:\n%s' % (search_string, cql)) + + # both + kwargs['initial_condition'] = ['init', 'cond'] + kwargs['final_func'] = 'List_As_String' + with self.VerifiedAggregate(self, **kwargs) as va: + meta = self.keyspace_aggregate_meta[va.signature] + self.assertListEqual(meta.initial_condition, kwargs['initial_condition']) + self.assertEqual(meta.final_func, kwargs['final_func']) + cql = meta.as_cql_query() + init_cond_idx = cql.find("INITCOND %s" % Encoder().cql_encode_all_types(kwargs['initial_condition'])) + final_func_idx = cql.find('FINALFUNC "%s"' % kwargs['final_func']) + self.assertNotIn(-1, (init_cond_idx, final_func_idx)) + self.assertGreater(init_cond_idx, final_func_idx) From 51d0a53c4c05696dc3dba2e562b1ea63ada69e88 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Apr 2015 10:37:32 -0500 Subject: [PATCH 0030/2431] Remove link to disabled cqlengine issues --- docs/cqlengine/third_party.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/cqlengine/third_party.rst b/docs/cqlengine/third_party.rst index 89f23eccb8..c4c99dbf54 100644 --- a/docs/cqlengine/third_party.rst +++ b/docs/cqlengine/third_party.rst @@ -31,9 +31,6 @@ Here's how, in substance, CQLengine can be plugged to `Celery app = Celery() -For more details, see `issue #237 -`_. - uWSGI ----- From f5d5f905e7cb7db97b4c06cbd314728c2acf410b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Apr 2015 15:34:42 -0500 Subject: [PATCH 0031/2431] Removed unused parameters from PreparedStatement init --- cassandra/query.py | 9 ++------- tests/unit/test_parameter_binding.py | 9 ++++----- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cassandra/query.py b/cassandra/query.py index fc71b8f62b..031a1684e5 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -338,19 +338,14 @@ class PreparedStatement(object): fetch_size = FETCH_SIZE_UNSET - def __init__(self, column_metadata, query_id, routing_key_indexes, query, keyspace, - protocol_version, consistency_level=None, serial_consistency_level=None, - fetch_size=FETCH_SIZE_UNSET): + def __init__(self, column_metadata, query_id, routing_key_indexes, query, + keyspace, protocol_version): self.column_metadata = column_metadata self.query_id = query_id self.routing_key_indexes = routing_key_indexes self.query_string = query self.keyspace = keyspace self.protocol_version = protocol_version - self.consistency_level = consistency_level - self.serial_consistency_level = serial_consistency_level - if fetch_size is not FETCH_SIZE_UNSET: - self.fetch_size = fetch_size @classmethod def from_message(cls, query_id, column_metadata, pk_indexes, cluster_metadata, query, prepared_keyspace, protocol_version): diff --git a/tests/unit/test_parameter_binding.py b/tests/unit/test_parameter_binding.py index f649c5207c..3fce7b9e91 100644 --- a/tests/unit/test_parameter_binding.py +++ b/tests/unit/test_parameter_binding.py @@ -128,8 +128,8 @@ def test_inherit_fetch_size(self): routing_key_indexes=[], query=None, keyspace=keyspace, - protocol_version=2, - fetch_size=1234) + protocol_version=2) + prepared_statement.fetch_size = 1234 bound_statement = BoundStatement(prepared_statement=prepared_statement) self.assertEqual(1234, bound_statement.fetch_size) @@ -147,10 +147,9 @@ def test_too_few_parameters_for_key(self): routing_key_indexes=[0, 1], query=None, keyspace=keyspace, - protocol_version=2, - fetch_size=1234) + protocol_version=2) self.assertRaises(ValueError, prepared_statement.bind, (1,)) - bound = prepared_statement.bind((1,2)) + bound = prepared_statement.bind((1, 2)) self.assertEqual(bound.keyspace, keyspace) From 8772e753cdf06a49822a9cf0f9a3819faf7f890b Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 16 Apr 2015 14:55:26 -0700 Subject: [PATCH 0032/2431] Don't run collection index tests on C* < 2.1 --- tests/integration/standard/test_metadata.py | 46 ++++----------------- 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index e5b2eab3ce..0afc15b559 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -46,47 +46,16 @@ class SchemaMetadataTests(unittest.TestCase): def cfname(self): return self._testMethodName.lower() - @classmethod - def setup_class(cls): - cluster = Cluster(protocol_version=PROTOCOL_VERSION) - session = cluster.connect() - try: - results = session.execute("SELECT keyspace_name FROM system.schema_keyspaces") - existing_keyspaces = [row[0] for row in results] - if cls.ksname in existing_keyspaces: - session.execute("DROP KEYSPACE %s" % cls.ksname) - - session.execute( - """ - CREATE KEYSPACE %s - WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}; - """ % cls.ksname) - finally: - cluster.shutdown() - - @classmethod - def teardown_class(cls): - cluster = Cluster(['127.0.0.1'], - protocol_version=PROTOCOL_VERSION) - session = cluster.connect() - try: - session.execute("DROP KEYSPACE %s" % cls.ksname) - finally: - cluster.shutdown() - def setUp(self): - self.cluster = Cluster(['127.0.0.1'], - protocol_version=PROTOCOL_VERSION) + self._cass_version, self._cql_version = get_server_versions() + + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) self.session = self.cluster.connect() + self.session.execute("CREATE KEYSPACE schemametadatatest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}") def tearDown(self): - try: - self.session.execute( - """ - DROP TABLE {ksname}.{cfname} - """.format(ksname=self.ksname, cfname=self.cfname)) - finally: - self.cluster.shutdown() + self.session.execute("DROP KEYSPACE schemametadatatest") + self.cluster.shutdown() def make_create_statement(self, partition_cols, clustering_cols=None, other_cols=None, compact=False): clustering_cols = clustering_cols or [] @@ -309,6 +278,9 @@ def test_indexes(self): self.assertIn('CREATE INDEX e_index', statement) def test_collection_indexes(self): + if get_server_versions()[0] < (2, 1, 0): + raise unittest.SkipTest("Secondary index on collections were introduced in Cassandra 2.1") + self.session.execute("CREATE TABLE %s.%s (a int PRIMARY KEY, b map)" % (self.ksname, self.cfname)) self.session.execute("CREATE INDEX index1 ON %s.%s (keys(b))" From 41d9394c077dccba4146d334b675a22fd4a682df Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 16 Apr 2015 15:25:14 -0700 Subject: [PATCH 0033/2431] Don't change blob params for assertion in C* 1.2 --- tests/integration/standard/test_types.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index b256e87595..c4821d9c87 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -57,7 +57,7 @@ def tearDown(self): def test_can_insert_blob_type_as_string(self): """ - Tests that blob type in Cassandra does not map to string in Python + Tests that byte strings in Python maps to blob type in Cassandra """ c = Cluster(protocol_version=PROTOCOL_VERSION) @@ -68,9 +68,7 @@ def test_can_insert_blob_type_as_string(self): params = ['key1', b'blobyblob'] query = "INSERT INTO blobstring (a, b) VALUES (%s, %s)" - # In python 3, the 'bytes' type is treated as a blob, so we can - # correctly encode it with hex notation. - # In python2, we don't treat the 'str' type as a blob, so we'll encode it + # In python2, with Cassandra > 2.0, we don't treat the 'byte str' type as a blob, so we'll encode it # as a string literal and have the following failure. if six.PY2 and self._cql_version >= (3, 1, 0): # Blob values can't be specified using string notation in CQL 3.1.0 and @@ -81,10 +79,14 @@ def test_can_insert_blob_type_as_string(self): msg = r'.*Invalid STRING constant \(.*?\) for b of type blob.*' self.assertRaisesRegexp(InvalidRequest, msg, s.execute, query, params) return - elif six.PY2: - params[1] = params[1].encode('hex') - s.execute(query, params) + # In python2, with Cassandra < 2.0, we can manually encode the 'byte str' type as hex for insertion in a blob. + if six.PY2: + cass_params = [params[0], params[1].encode('hex')] + s.execute(query, cass_params) + # In python 3, the 'bytes' type is treated as a blob, so we can correctly encode it with hex notation. + else: + s.execute(query, params) results = s.execute("SELECT * FROM blobstring")[0] for expected, actual in zip(params, results): From aa2be65810e0aa2675e760df83296a366fcc094b Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 16 Apr 2015 15:41:58 -0700 Subject: [PATCH 0034/2431] Remove operation timeouts for write timeout metrics test --- tests/integration/standard/test_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/standard/test_metrics.py b/tests/integration/standard/test_metrics.py index 1996495c0b..7b19404907 100644 --- a/tests/integration/standard/test_metrics.py +++ b/tests/integration/standard/test_metrics.py @@ -86,7 +86,7 @@ def test_write_timeout(self): # Test write query = SimpleStatement("INSERT INTO test (k, v) VALUES (2, 2)", consistency_level=ConsistencyLevel.ALL) with self.assertRaises(WriteTimeout): - session.execute(query) + session.execute(query, timeout=None) self.assertEqual(1, cluster.metrics.stats.write_timeouts) finally: From b2514a898a05b93a9085ea7c7528e5a425b85c86 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 16 Apr 2015 17:53:28 -0700 Subject: [PATCH 0035/2431] Give extra time for auth setup on C* 1.2 tests --- tests/integration/standard/test_authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/standard/test_authentication.py b/tests/integration/standard/test_authentication.py index c698c986e2..3820ef7a90 100644 --- a/tests/integration/standard/test_authentication.py +++ b/tests/integration/standard/test_authentication.py @@ -40,7 +40,7 @@ def setup_module(): ccm_cluster.start(wait_for_binary_proto=True, wait_other_notice=True) # there seems to be some race, with some versions of C* taking longer to # get the auth (and default user) setup. Sleep here to give it a chance - time.sleep(2) + time.sleep(10) def teardown_module(): From 80e952b298cc26ff3199e320b7d97626eff1f5e9 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 17 Apr 2015 10:22:37 -0500 Subject: [PATCH 0036/2431] Custom payloads for protocol v4 PYTHON-280 --- CHANGELOG.rst | 1 - cassandra/auth.py | 2 ++ cassandra/cluster.py | 42 ++++++++++++++++++++++++----- cassandra/protocol.py | 43 +++++++++++++++++++++++++++++- cassandra/query.py | 61 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 128 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af2d611309..8f91416846 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -78,7 +78,6 @@ Bug Fixes --------- * Make execute_concurrent compatible with Python 2.6 (PYTHON-159) * Handle Unauthorized message on schema_triggers query (PYTHON-155) -* Make execute_concurrent compatible with Python 2.6 (github-197) * Pure Python sorted set in support of UDTs nested in collections (PYTON-167) * Support CUSTOM index metadata and string export (PYTHON-165) diff --git a/cassandra/auth.py b/cassandra/auth.py index 67d302a9e8..508bd150ef 100644 --- a/cassandra/auth.py +++ b/cassandra/auth.py @@ -15,6 +15,7 @@ except ImportError: SASLClient = None + class AuthProvider(object): """ An abstract class that defines the interface that will be used for @@ -157,6 +158,7 @@ def __init__(self, **sasl_kwargs): def new_authenticator(self, host): return SaslAuthenticator(**self.sasl_kwargs) + class SaslAuthenticator(Authenticator): """ A pass-through :class:`~.Authenticator` using the third party package diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cb04152a5b..e84afd2ef1 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1361,7 +1361,7 @@ def __init__(self, cluster, hosts): for future in futures: future.result() - def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): + def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False, custom_payload=None): """ Execute the given query and synchronously wait for the response. @@ -1389,6 +1389,10 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): instance and not just a string. If there is an error fetching the trace details, the :attr:`~.Statement.trace` attribute will be left as :const:`None`. + + `custom_payload` is a dict as described in TODO section. If `query` is a Statement + with its own custom_payload. the message will be a union of the two, + with the values specified here taking precedence. """ if timeout is _NOT_SET: timeout = self.default_timeout @@ -1398,7 +1402,7 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): "The query argument must be an instance of a subclass of " "cassandra.query.Statement when trace=True") - future = self.execute_async(query, parameters, trace) + future = self.execute_async(query, parameters, trace, custom_payload) try: result = future.result(timeout) finally: @@ -1410,7 +1414,7 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): return result - def execute_async(self, query, parameters=None, trace=False): + def execute_async(self, query, parameters=None, trace=False, custom_payload=None): """ Execute the given query and return a :class:`~.ResponseFuture` object which callbacks may be attached to for asynchronous response @@ -1422,6 +1426,13 @@ def execute_async(self, query, parameters=None, trace=False): :meth:`.ResponseFuture.get_query_trace()` after the request completes to retrieve a :class:`.QueryTrace` instance. + `custom_payload` is a dict as described in TODO section. If `query` is + a Statement with a custom_payload specified. the message will be a + union of the two, with the values specified here taking precedence. + + If the server sends a custom payload in the response message, + the dict can be obtained via :attr:`.ResponseFuture.custom_payload` + Example usage:: >>> session = cluster.connect() @@ -1447,11 +1458,11 @@ def execute_async(self, query, parameters=None, trace=False): ... log.exception("Operation failed:") """ - future = self._create_response_future(query, parameters, trace) + future = self._create_response_future(query, parameters, trace, custom_payload) future.send_request() return future - def _create_response_future(self, query, parameters, trace): + def _create_response_future(self, query, parameters, trace, custom_payload): """ Returns the ResponseFuture before calling send_request() on it """ prepared_statement = None @@ -1501,13 +1512,16 @@ def _create_response_future(self, query, parameters, trace): if trace: message.tracing = True + message.update_custom_payload(query.custom_payload) + message.update_custom_payload(custom_payload) + return ResponseFuture( self, message, query, self.default_timeout, metrics=self._metrics, prepared_statement=prepared_statement) - def prepare(self, query): + def prepare(self, query, custom_payload=None): """ - Prepares a query string, returing a :class:`~cassandra.query.PreparedStatement` + Prepares a query string, returning a :class:`~cassandra.query.PreparedStatement` instance which can be used as follows:: >>> session = cluster.connect("mykeyspace") @@ -1530,8 +1544,12 @@ def prepare(self, query): **Important**: PreparedStatements should be prepared only once. Preparing the same query more than once will likely affect performance. + + `custom_payload` is a key value map to be passed along with the prepare + message. See TODO: refer to doc section """ message = PrepareMessage(query=query) + message.custom_payload = custom_payload future = ResponseFuture(self, message, query=None) try: future.send_request() @@ -1543,6 +1561,7 @@ def prepare(self, query): prepared_statement = PreparedStatement.from_message( query_id, column_metadata, pk_indexes, self.cluster.metadata, query, self.keyspace, self._protocol_version) + prepared_statement.custom_payload = future.custom_payload host = future._current_host try: @@ -2567,6 +2586,7 @@ class ResponseFuture(object): _start_time = None _metrics = None _paging_state = None + _custom_payload = None def __init__(self, session, message, query, default_timeout=None, metrics=None, prepared_statement=None): self.session = session @@ -2654,6 +2674,12 @@ def has_more_pages(self): """ return self._paging_state is not None + @property + def custom_payload(self): + if not self._event.is_set(): + raise Exception("custom_payload cannot be retrieved before ResponseFuture is finalized") + return self._custom_payload + def start_fetching_next_page(self): """ If there are more pages left in the query result, this asynchronously @@ -2690,6 +2716,8 @@ def _set_result(self, response): if trace_id: self._query_trace = QueryTrace(trace_id, self.session) + self._custom_payload = getattr(response, 'custom_payload', None) + if isinstance(response, ResultMessage): if response.kind == RESULT_KIND_SET_KEYSPACE: session = getattr(self, 'session', None) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index d81a650731..a48af3485c 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -54,6 +54,7 @@ class InternalError(Exception): COMPRESSED_FLAG = 0x01 TRACING_FLAG = 0x02 +CUSTOM_PAYLOAD_FLAG = 0x04 _message_types_by_name = {} _message_types_by_opcode = {} @@ -70,13 +71,19 @@ def __init__(cls, name, bases, dct): class _MessageType(object): tracing = False + custom_payload = None def to_binary(self, stream_id, protocol_version, compression=None): + flags = 0 body = io.BytesIO() + if self.custom_payload: + if protocol_version < 4: + raise UnsupportedOperation("Custom key/value payloads can only be used with protocol version 4 or higher") + flags |= CUSTOM_PAYLOAD_FLAG + write_bytesmap(body, self.custom_payload) self.send_body(body, protocol_version) body = body.getvalue() - flags = 0 if compression and len(body) > 0: body = compression(body) flags |= COMPRESSED_FLAG @@ -89,6 +96,12 @@ def to_binary(self, stream_id, protocol_version, compression=None): return msg.getvalue() + def update_custom_payload(self, other): + if other: + if not self.custom_payload: + self.custom_payload = {} + self.custom_payload.update(other) + def __repr__(self): return '<%s(%s)>' % (self.__class__.__name__, ', '.join('%s=%r' % i for i in _get_params(self))) @@ -116,6 +129,12 @@ def decode_response(protocol_version, user_type_map, stream_id, flags, opcode, b else: trace_id = None + if flags & CUSTOM_PAYLOAD_FLAG: + custom_payload = read_bytesmap(body) + flags ^= CUSTOM_PAYLOAD_FLAG + else: + custom_payload = None + if flags: log.warning("Unknown protocol flags set: %02x. May cause problems.", flags) @@ -123,6 +142,7 @@ def decode_response(protocol_version, user_type_map, stream_id, flags, opcode, b msg = msg_class.recv_body(body, protocol_version, user_type_map) msg.stream_id = stream_id msg.trace_id = trace_id + msg.custom_payload = custom_payload return msg @@ -918,6 +938,11 @@ def read_binary_string(f): return contents +def write_binary_string(f, s): + write_short(f, len(s)) + f.write(s) + + def write_string(f, s): if isinstance(s, six.text_type): s = s.encode('utf8') @@ -969,6 +994,22 @@ def write_stringmap(f, strmap): write_string(f, v) +def read_bytesmap(f): + numpairs = read_short(f) + bytesmap = {} + for _ in range(numpairs): + k = read_string(f) + bytesmap[k] = read_binary_string(f) + return bytesmap + + +def write_bytesmap(f, bytesmap): + write_short(f, len(bytesmap)) + for k, v in bytesmap.items(): + write_string(f, k) + write_binary_string(f, v) + + def read_stringmultimap(f): numkeys = read_short(f) strmmap = {} diff --git a/cassandra/query.py b/cassandra/query.py index 031a1684e5..70d16e9e40 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -197,11 +197,25 @@ class Statement(object): .. versionadded:: 2.1.3 """ + custom_payload = None + """ + TODO: refer to custom proto doc section + A string:binary_type dict holding custom key/value pairs to be passed + in the frame to a custom QueryHandler on the server side. + + By default these values are ignored by the server. + + These are only allowed when using protocol version 4 or higher. + + .. versionadded:: 3.0.0 + """ + _serial_consistency_level = None _routing_key = None def __init__(self, retry_policy=None, consistency_level=None, routing_key=None, - serial_consistency_level=None, fetch_size=FETCH_SIZE_UNSET, keyspace=None): + serial_consistency_level=None, fetch_size=FETCH_SIZE_UNSET, keyspace=None, + custom_payload=None): self.retry_policy = retry_policy if consistency_level is not None: self.consistency_level = consistency_level @@ -212,6 +226,8 @@ def __init__(self, retry_policy=None, consistency_level=None, routing_key=None, self.fetch_size = fetch_size if keyspace is not None: self.keyspace = keyspace + if custom_payload is not None: + self.custom_payload = custom_payload def _get_routing_key(self): return self._routing_key @@ -290,8 +306,7 @@ def _del_serial_consistency_level(self): class SimpleStatement(Statement): """ - A simple, un-prepared query. All attributes of :class:`Statement` apply - to this class as well. + A simple, un-prepared query. """ def __init__(self, query_string, *args, **kwargs): @@ -299,6 +314,8 @@ def __init__(self, query_string, *args, **kwargs): `query_string` should be a literal CQL statement with the exception of parameter placeholders that will be filled through the `parameters` argument of :meth:`.Session.execute()`. + + All arguments to :class:`Statement` apply to this class as well """ Statement.__init__(self, *args, **kwargs) self._query_string = query_string @@ -338,6 +355,8 @@ class PreparedStatement(object): fetch_size = FETCH_SIZE_UNSET + custom_payload = None + def __init__(self, column_metadata, query_id, routing_key_indexes, query, keyspace, protocol_version): self.column_metadata = column_metadata @@ -397,8 +416,6 @@ class BoundStatement(Statement): """ A prepared statement that has been bound to a particular set of values. These may be created directly or through :meth:`.PreparedStatement.bind()`. - - All attributes of :class:`Statement` apply to this class as well. """ prepared_statement = None @@ -414,13 +431,15 @@ class BoundStatement(Statement): def __init__(self, prepared_statement, *args, **kwargs): """ `prepared_statement` should be an instance of :class:`PreparedStatement`. - All other ``*args`` and ``**kwargs`` will be passed to :class:`.Statement`. + + All arguments to :class:`Statement` apply to this class as well """ self.prepared_statement = prepared_statement self.consistency_level = prepared_statement.consistency_level self.serial_consistency_level = prepared_statement.serial_consistency_level self.fetch_size = prepared_statement.fetch_size + self.custom_payload = prepared_statement.custom_payload self.values = [] meta = prepared_statement.column_metadata @@ -601,7 +620,8 @@ class BatchStatement(Statement): _session = None def __init__(self, batch_type=BatchType.LOGGED, retry_policy=None, - consistency_level=None, serial_consistency_level=None, session=None): + consistency_level=None, serial_consistency_level=None, + session=None, custom_payload=None): """ `batch_type` specifies The :class:`.BatchType` for the batch operation. Defaults to :attr:`.BatchType.LOGGED`. @@ -612,6 +632,10 @@ def __init__(self, batch_type=BatchType.LOGGED, retry_policy=None, `consistency_level` should be a :class:`~.ConsistencyLevel` value to be used for all operations in the batch. + `custom_payload` is a key-value map TODO: refer to doc section + Note: as Statement objects are added to the batch, this map is + updated with values from their custom payloads. + Example usage: .. code-block:: python @@ -637,12 +661,15 @@ def __init__(self, batch_type=BatchType.LOGGED, retry_policy=None, .. versionchanged:: 2.1.0 Added `serial_consistency_level` as a parameter + + .. versionchanged:: 3.0.0 + Added `custom_payload` as a parameter """ self.batch_type = batch_type self._statements_and_parameters = [] self._session = session Statement.__init__(self, retry_policy=retry_policy, consistency_level=consistency_level, - serial_consistency_level=serial_consistency_level) + serial_consistency_level=serial_consistency_level, custom_payload=custom_payload) def add(self, statement, parameters=None): """ @@ -660,7 +687,7 @@ def add(self, statement, parameters=None): elif isinstance(statement, PreparedStatement): query_id = statement.query_id bound_statement = statement.bind(() if parameters is None else parameters) - self._maybe_set_routing_attributes(bound_statement) + self._update_state(bound_statement) self._statements_and_parameters.append( (True, query_id, bound_statement.values)) elif isinstance(statement, BoundStatement): @@ -668,7 +695,7 @@ def add(self, statement, parameters=None): raise ValueError( "Parameters cannot be passed with a BoundStatement " "to BatchStatement.add()") - self._maybe_set_routing_attributes(statement) + self._update_state(statement) self._statements_and_parameters.append( (True, statement.prepared_statement.query_id, statement.values)) else: @@ -677,7 +704,7 @@ def add(self, statement, parameters=None): if parameters: encoder = Encoder() if self._session is None else self._session.encoder query_string = bind_params(query_string, parameters, encoder) - self._maybe_set_routing_attributes(statement) + self._update_state(statement) self._statements_and_parameters.append((False, query_string, ())) return self @@ -696,6 +723,16 @@ def _maybe_set_routing_attributes(self, statement): self.routing_key = statement.routing_key self.keyspace = statement.keyspace + def _update_custom_payload(self, statement): + if statement.custom_payload: + if self.custom_payload is None: + self.custom_payload = {} + self.custom_payload.update(statement.custom_payload) + + def _update_state(self, statement): + self._maybe_set_routing_attributes(statement) + self._update_custom_payload(statement) + def __str__(self): consistency = ConsistencyLevel.value_to_name.get(self.consistency_level, 'Not Set') return (u'' % @@ -836,7 +873,7 @@ def populate(self, max_wait=2.0): def _execute(self, query, parameters, time_spent, max_wait): # in case the user switched the row factory, set it to namedtuple for this query - future = self._session._create_response_future(query, parameters, trace=False) + future = self._session._create_response_future(query, parameters, trace=False, custom_payload=None) future.row_factory = named_tuple_factory future.send_request() From 21ad7ad679d1e6be3e2cd0701881e25ae74af505 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 17 Apr 2015 14:34:21 -0700 Subject: [PATCH 0037/2431] Fixed IPv6 tests, and streamlined cluster removal --- tests/integration/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index ce3f042bf6..4b3c538145 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -26,6 +26,7 @@ import os from threading import Event import six +from subprocess import call from itertools import groupby @@ -201,11 +202,13 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True): if start: log.debug("Starting ccm %s cluster", cluster_name) cluster.start(wait_for_binary_proto=True, wait_other_notice=True) - setup_test_keyspace() + setup_test_keyspace(ipformat=ipformat) CCM_CLUSTER = cluster except Exception: - log.exception("Failed to start ccm cluster:") + log.exception("Failed to start ccm cluster. Removing cluster.") + remove_cluster() + call(["pkill", "-9", "-f", ".ccm"]) raise @@ -228,11 +231,14 @@ def teardown_package(): log.warn('Did not find cluster: %s' % cluster_name) -def setup_test_keyspace(): +def setup_test_keyspace(ipformat=None): # wait for nodes to startup time.sleep(10) - cluster = Cluster(protocol_version=PROTOCOL_VERSION) + if not ipformat: + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + else: + cluster = Cluster(contact_points=["::1"], protocol_version=PROTOCOL_VERSION) session = cluster.connect() try: From f4191903b7e85e82849c20401d5e776bf45d2a9e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 17 Apr 2015 16:51:52 -0500 Subject: [PATCH 0038/2431] Docs for custom_payload --- cassandra/cluster.py | 17 +++++++++-------- cassandra/query.py | 11 ++++------- docs/api/index.rst | 1 + 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index e84afd2ef1..28885ae9aa 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1390,9 +1390,9 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False, custom_ trace details, the :attr:`~.Statement.trace` attribute will be left as :const:`None`. - `custom_payload` is a dict as described in TODO section. If `query` is a Statement - with its own custom_payload. the message will be a union of the two, - with the values specified here taking precedence. + `custom_payload` is a :ref:`custom_payload` dict to be passed to the server. + If `query` is a Statement with its own custom_payload. The message payload + will be a union of the two, with the values specified here taking precedence. """ if timeout is _NOT_SET: timeout = self.default_timeout @@ -1426,12 +1426,13 @@ def execute_async(self, query, parameters=None, trace=False, custom_payload=None :meth:`.ResponseFuture.get_query_trace()` after the request completes to retrieve a :class:`.QueryTrace` instance. - `custom_payload` is a dict as described in TODO section. If `query` is - a Statement with a custom_payload specified. the message will be a - union of the two, with the values specified here taking precedence. + `custom_payload` is a :ref:`custom_payload` dict to be passed to the server. + If `query` is a Statement with its own custom_payload. The message payload + will be a union of the two, with the values specified here taking precedence. If the server sends a custom payload in the response message, - the dict can be obtained via :attr:`.ResponseFuture.custom_payload` + the dict can be obtained following :meth:`.ResponseFuture.result` via + :attr:`.ResponseFuture.custom_payload` Example usage:: @@ -1546,7 +1547,7 @@ def prepare(self, query, custom_payload=None): Preparing the same query more than once will likely affect performance. `custom_payload` is a key value map to be passed along with the prepare - message. See TODO: refer to doc section + message. See :ref:`custom_payload`. """ message = PrepareMessage(query=query) message.custom_payload = custom_payload diff --git a/cassandra/query.py b/cassandra/query.py index 70d16e9e40..722e0d59c1 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -199,11 +199,7 @@ class Statement(object): custom_payload = None """ - TODO: refer to custom proto doc section - A string:binary_type dict holding custom key/value pairs to be passed - in the frame to a custom QueryHandler on the server side. - - By default these values are ignored by the server. + :ref:`custom_payload` to be passed to the server. These are only allowed when using protocol version 4 or higher. @@ -632,9 +628,10 @@ def __init__(self, batch_type=BatchType.LOGGED, retry_policy=None, `consistency_level` should be a :class:`~.ConsistencyLevel` value to be used for all operations in the batch. - `custom_payload` is a key-value map TODO: refer to doc section + `custom_payload` is a :ref:`custom_payload` passed to the server. Note: as Statement objects are added to the batch, this map is - updated with values from their custom payloads. + updated with any values found in their custom payloads. These are + only allowed when using protocol version 4 or higher. Example usage: diff --git a/docs/api/index.rst b/docs/api/index.rst index e0fe9810d2..340a5e0235 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -14,6 +14,7 @@ Core Driver cassandra/metrics cassandra/query cassandra/pool + cassandra/protocol cassandra/encoder cassandra/decoder cassandra/concurrent From 567f9b0216f51c9e80ea7d8ec5af96ba6acf1850 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Apr 2015 10:08:20 -0500 Subject: [PATCH 0039/2431] Fix variable rename bug Peer reveiw input --- cassandra/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index b6e0aee7b3..a8e3ba9241 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -161,7 +161,7 @@ def rebuild_schema(self, ks_results, type_results, function_results, # remove not-just-added keyspaces removed_keyspaces = [name for name in self.keyspaces.keys() - if ksname not in current_keyspaces] + if name not in current_keyspaces] self.keyspaces = dict((name, meta) for name, meta in self.keyspaces.items() if name in current_keyspaces) for ksname in removed_keyspaces: From 2c062213fde00a0c30b3dcbddd2e4dea6d7c210e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Apr 2015 14:22:09 -0500 Subject: [PATCH 0040/2431] Add the missing protocol doc file --- docs/api/cassandra/protocol.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/api/cassandra/protocol.rst diff --git a/docs/api/cassandra/protocol.rst b/docs/api/cassandra/protocol.rst new file mode 100644 index 0000000000..a6b4d1c41a --- /dev/null +++ b/docs/api/cassandra/protocol.rst @@ -0,0 +1,11 @@ +.. _custom_payload: + +Custom Payload +============== +Native protocol version 4+ allows for a custom payload to be sent between clients +and custom query handlers. The payload is specified as a string:binary_type dict +holding custom key/value pairs. + +By default these are ignored by the server. They can be useful for servers implementing +a custom QueryHandler. + From 35c6ce5365345571832028d5bdb97efd1a4a0435 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Apr 2015 15:58:44 -0500 Subject: [PATCH 0041/2431] Refactor redundant Connection factory methods to base --- cassandra/connection.py | 18 ++++++++++++++++++ cassandra/io/asyncorereactor.py | 13 ------------- cassandra/io/eventletreactor.py | 13 ------------- cassandra/io/geventreactor.py | 13 ------------- cassandra/io/libevreactor.py | 13 ------------- cassandra/io/twistedreactor.py | 18 ------------------ 6 files changed, 18 insertions(+), 70 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 2a020c0615..6cbd434650 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -239,6 +239,24 @@ def handle_fork(self): """ pass + @classmethod + def factory(cls, *args, **kwargs): + """ + A factory function which returns connections which have + succeeded in connecting and are ready for service (or + raises an exception otherwise). + """ + timeout = kwargs.pop('timeout', 5.0) + conn = cls(*args, **kwargs) + conn.connected_event.wait(timeout) + if conn.last_error: + raise conn.last_error + elif not conn.connected_event.is_set(): + conn.close() + raise OperationTimedOut("Timed out creating connection") + else: + return conn + def close(self): raise NotImplementedError() diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 2ac7156d61..ef687c388c 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -156,19 +156,6 @@ def handle_fork(cls): cls._loop._cleanup() cls._loop = None - @classmethod - def factory(cls, *args, **kwargs): - timeout = kwargs.pop('timeout', 5.0) - conn = cls(*args, **kwargs) - conn.connected_event.wait(timeout) - if conn.last_error: - raise conn.last_error - elif not conn.connected_event.is_set(): - conn.close() - raise OperationTimedOut("Timed out creating connection") - else: - return conn - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) asyncore.dispatcher.__init__(self) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index ceac6a951e..670d0f1865 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -57,19 +57,6 @@ class EventletConnection(Connection): def initialize_reactor(cls): eventlet.monkey_patch() - @classmethod - def factory(cls, *args, **kwargs): - timeout = kwargs.pop('timeout', 5.0) - conn = cls(*args, **kwargs) - conn.connected_event.wait(timeout) - if conn.last_error: - raise conn.last_error - elif not conn.connected_event.is_set(): - conn.close() - raise OperationTimedOut("Timed out creating connection") - else: - return conn - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 4cd9c68109..6e9af0da4d 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -50,19 +50,6 @@ class GeventConnection(Connection): _write_watcher = None _socket = None - @classmethod - def factory(cls, *args, **kwargs): - timeout = kwargs.pop('timeout', 5.0) - conn = cls(*args, **kwargs) - conn.connected_event.wait(timeout) - if conn.last_error: - raise conn.last_error - elif not conn.connected_event.is_set(): - conn.close() - raise OperationTimedOut("Timed out creating connection") - else: - return conn - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index db11eaf8be..93b4c97854 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -236,19 +236,6 @@ def handle_fork(cls): cls._libevloop._cleanup() cls._libevloop = None - @classmethod - def factory(cls, *args, **kwargs): - timeout = kwargs.pop('timeout', 5.0) - conn = cls(*args, **kwargs) - conn.connected_event.wait(timeout) - if conn.last_error: - raise conn.last_error - elif not conn.connected_event.is_set(): - conn.close() - raise OperationTimedOut("Timed out creating new connection") - else: - return conn - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index 1a5a64e796..ff81e5613f 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -148,24 +148,6 @@ def initialize_reactor(cls): if not cls._loop: cls._loop = TwistedLoop() - @classmethod - def factory(cls, *args, **kwargs): - """ - A factory function which returns connections which have - succeeded in connecting and are ready for service (or - raises an exception otherwise). - """ - timeout = kwargs.pop('timeout', 5.0) - conn = cls(*args, **kwargs) - conn.connected_event.wait(timeout) - if conn.last_error: - raise conn.last_error - elif not conn.connected_event.is_set(): - conn.close() - raise OperationTimedOut("Timed out creating connection") - else: - return conn - def __init__(self, *args, **kwargs): """ Initialization method. From 833feeb9a06c00fb5afb1f80d6880799a5ec451f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Apr 2015 15:31:48 -0500 Subject: [PATCH 0042/2431] Deprecate cqlengine.Float(double_precision=True) Overload deprecated in favor of distinct Float, Double types PYTHON-246 --- cassandra/cqlengine/columns.py | 38 ++++++++++----- docs/api/cassandra/cqlengine/columns.rst | 2 + .../cqlengine/columns/test_value_io.py | 47 +++++++++++++++---- .../integration/cqlengine/model/test_udts.py | 4 +- 4 files changed, 69 insertions(+), 22 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index ec6a468f60..ae17779590 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -591,18 +591,9 @@ def to_python(self, value): return self.validate(value) -class Float(Column): - """ - Stores a floating point value - """ - db_type = 'double' - - def __init__(self, double_precision=True, **kwargs): - self.db_type = 'double' if double_precision else 'float' - super(Float, self).__init__(**kwargs) - +class BaseFloat(Column): def validate(self, value): - value = super(Float, self).validate(value) + value = super(BaseFloat, self).validate(value) if value is None: return try: @@ -617,6 +608,31 @@ def to_database(self, value): return self.validate(value) +class Float(BaseFloat): + """ + Stores a single-precision floating-point value + """ + db_type = 'float' + + def __init__(self, double_precision=None, **kwargs): + if double_precision is None or bool(double_precision): + msg = "Float(double_precision=True) is deprecated. Use Double() type instead." + double_precision = True + warnings.warn(msg, DeprecationWarning) + log.warning(msg) + + self.db_type = 'double' if double_precision else 'float' + + super(Float, self).__init__(**kwargs) + + +class Double(BaseFloat): + """ + Stores a double-precision floating-point value + """ + db_type = 'double' + + class Decimal(Column): """ Stores a variable precision decimal value diff --git a/docs/api/cassandra/cqlengine/columns.rst b/docs/api/cassandra/cqlengine/columns.rst index 425e01e03a..a214d2cd94 100644 --- a/docs/api/cassandra/cqlengine/columns.rst +++ b/docs/api/cassandra/cqlengine/columns.rst @@ -58,6 +58,8 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: Decimal(**kwargs) +.. autoclass:: Double + .. autoclass:: Float .. autoclass:: Integer(**kwargs) diff --git a/tests/integration/cqlengine/columns/test_value_io.py b/tests/integration/cqlengine/columns/test_value_io.py index 2ebfa89b34..04be247a84 100644 --- a/tests/integration/cqlengine/columns/test_value_io.py +++ b/tests/integration/cqlengine/columns/test_value_io.py @@ -50,8 +50,9 @@ class BaseColumnIOTest(BaseCassEngTestCase): def setUpClass(cls): super(BaseColumnIOTest, cls).setUpClass() - #if the test column hasn't been defined, bail out - if not cls.column: return + # if the test column hasn't been defined, bail out + if not cls.column: + return # create a table with the given column class IOTestModel(Model): @@ -62,7 +63,7 @@ class IOTestModel(Model): cls._generated_model = IOTestModel sync_table(cls._generated_model) - #tupleify the tested values + # tupleify the tested values if not isinstance(cls.pkey_val, tuple): cls.pkey_val = cls.pkey_val, if not isinstance(cls.data_val, tuple): @@ -71,7 +72,8 @@ class IOTestModel(Model): @classmethod def tearDownClass(cls): super(BaseColumnIOTest, cls).tearDownClass() - if not cls.column: return + if not cls.column: + return drop_table(cls._generated_model) def comparator_converter(self, val): @@ -80,31 +82,35 @@ def comparator_converter(self, val): def test_column_io(self): """ Tests the given models class creates and retrieves values as expected """ - if not self.column: return + if not self.column: + return for pkey, data in zip(self.pkey_val, self.data_val): - #create + # create m1 = self._generated_model.create(pkey=pkey, data=data) - #get + # get m2 = self._generated_model.get(pkey=pkey) assert m1.pkey == m2.pkey == self.comparator_converter(pkey), self.column assert m1.data == m2.data == self.comparator_converter(data), self.column - #delete + # delete self._generated_model.filter(pkey=pkey).delete() + class TestBlobIO(BaseColumnIOTest): column = columns.Blob pkey_val = six.b('blake'), uuid4().bytes data_val = six.b('eggleston'), uuid4().bytes + class TestBlobIO2(BaseColumnIOTest): column = columns.Blob pkey_val = bytearray(six.b('blake')), uuid4().bytes data_val = bytearray(six.b('eggleston')), uuid4().bytes + class TestTextIO(BaseColumnIOTest): column = columns.Text @@ -118,18 +124,21 @@ class TestNonBinaryTextIO(BaseColumnIOTest): pkey_val = 'bacon' data_val = '0xmonkey' + class TestInteger(BaseColumnIOTest): column = columns.Integer pkey_val = 5 data_val = 6 + class TestBigInt(BaseColumnIOTest): column = columns.BigInt pkey_val = 6 data_val = pow(2, 63) - 1 + class TestDateTime(BaseColumnIOTest): column = columns.DateTime @@ -138,6 +147,7 @@ class TestDateTime(BaseColumnIOTest): pkey_val = now data_val = now + timedelta(days=1) + class TestDate(BaseColumnIOTest): column = columns.Date @@ -146,6 +156,7 @@ class TestDate(BaseColumnIOTest): pkey_val = now data_val = now + timedelta(days=1) + class TestUUID(BaseColumnIOTest): column = columns.UUID @@ -156,6 +167,7 @@ class TestUUID(BaseColumnIOTest): def comparator_converter(self, val): return val if isinstance(val, UUID) else UUID(val) + class TestTimeUUID(BaseColumnIOTest): column = columns.TimeUUID @@ -166,13 +178,29 @@ class TestTimeUUID(BaseColumnIOTest): def comparator_converter(self, val): return val if isinstance(val, UUID) else UUID(val) + +# until Floats are implicitly single: +class FloatSingle(columns.Float): + def __init__(self, **kwargs): + super(FloatSingle, self).__init__(double_precision=False, **kwargs) + + class TestFloatIO(BaseColumnIOTest): - column = columns.Float + column = FloatSingle + + pkey_val = 4.75 + data_val = -1.5 + + +class TestDoubleIO(BaseColumnIOTest): + + column = columns.Double pkey_val = 3.14 data_val = -1982.11 + class TestDecimalIO(BaseColumnIOTest): column = columns.Decimal @@ -183,6 +211,7 @@ class TestDecimalIO(BaseColumnIOTest): def comparator_converter(self, val): return Decimal(val) + class TestQuoter(unittest.TestCase): def test_equals(self): diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index ab9386b8b4..89a55d1730 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -196,7 +196,7 @@ class AllDatatypes(UserType): e = columns.Date() f = columns.DateTime() g = columns.Decimal() - h = columns.Float() + h = columns.Float(double_precision=False) i = columns.Inet() j = columns.Integer() k = columns.Text() @@ -227,7 +227,7 @@ class AllDatatypes(UserType): e = columns.Date() f = columns.DateTime() g = columns.Decimal() - h = columns.Float(double_precision=True) + h = columns.Double() i = columns.Inet() j = columns.Integer() k = columns.Text() From 67eaa36776018e453042ce81f9c3645235fd9ca8 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Mon, 20 Apr 2015 16:41:47 -0700 Subject: [PATCH 0043/2431] Catch OperationTimedOut exception, removed sleeps --- tests/integration/long/test_consistency.py | 28 +++++++++++-------- .../long/test_loadbalancingpolicies.py | 26 +++++++++++++---- tests/integration/long/utils.py | 10 ++----- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/tests/integration/long/test_consistency.py b/tests/integration/long/test_consistency.py index de3b9fd92b..bf0cf7b937 100644 --- a/tests/integration/long/test_consistency.py +++ b/tests/integration/long/test_consistency.py @@ -12,25 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import struct -import traceback +import struct, logging, sys, traceback -import cassandra -from cassandra import ConsistencyLevel +from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, Unavailable from cassandra.cluster import Cluster -from cassandra.policies import TokenAwarePolicy, RoundRobinPolicy, \ - DowngradingConsistencyRetryPolicy +from cassandra.policies import TokenAwarePolicy, RoundRobinPolicy, DowngradingConsistencyRetryPolicy from cassandra.query import SimpleStatement from tests.integration import use_singledc, PROTOCOL_VERSION -from tests.integration.long.utils import force_stop, create_schema, \ - wait_for_down, wait_for_up, start, CoordinatorStats +from tests.integration.long.utils import (force_stop, create_schema, wait_for_down, wait_for_up, + start, CoordinatorStats) try: import unittest2 as unittest except ImportError: import unittest # noqa +log = logging.getLogger(__name__) + ALL_CONSISTENCY_LEVELS = set([ ConsistencyLevel.ANY, ConsistencyLevel.ONE, ConsistencyLevel.TWO, ConsistencyLevel.QUORUM, ConsistencyLevel.THREE, @@ -74,7 +73,14 @@ def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ON ss = SimpleStatement('SELECT * FROM cf WHERE k = 0', consistency_level=consistency_level, routing_key=routing_key) - self.coordinator_stats.add_coordinator(session.execute_async(ss)) + while True: + try: + self.coordinator_stats.add_coordinator(session.execute_async(ss)) + break + except (OperationTimedOut, ReadTimeout): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb def _assert_writes_succeed(self, session, keyspace, consistency_levels): for cl in consistency_levels: @@ -103,7 +109,7 @@ def _assert_writes_fail(self, session, keyspace, consistency_levels): try: self._insert(session, keyspace, 1, cl) self._cl_expected_failure(cl) - except (cassandra.Unavailable, cassandra.WriteTimeout): + except (Unavailable, WriteTimeout): pass def _assert_reads_fail(self, session, keyspace, consistency_levels): @@ -112,7 +118,7 @@ def _assert_reads_fail(self, session, keyspace, consistency_levels): try: self._query(session, keyspace, 1, cl) self._cl_expected_failure(cl) - except (cassandra.Unavailable, cassandra.ReadTimeout): + except (Unavailable, ReadTimeout): pass def _test_tokenaware_one_node_down(self, keyspace, rf, accepted): diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 6a9cb74c8e..cf5f3a191b 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import struct -import time -from cassandra import ConsistencyLevel, Unavailable +import struct, time, logging, sys, traceback + +from cassandra import ConsistencyLevel, Unavailable, OperationTimedOut, ReadTimeout from cassandra.cluster import Cluster, NoHostAvailable from cassandra.concurrent import execute_concurrent_with_args from cassandra.policies import (RoundRobinPolicy, DCAwareRoundRobinPolicy, @@ -32,6 +32,8 @@ except ImportError: import unittest # noqa +log = logging.getLogger(__name__) + class LoadBalancingPolicyTests(unittest.TestCase): @@ -59,14 +61,28 @@ def _query(self, session, keyspace, count=12, self.prepared = session.prepare(query_string) for i in range(count): - self.coordinator_stats.add_coordinator(session.execute_async(self.prepared.bind((0,)))) + while True: + try: + self.coordinator_stats.add_coordinator(session.execute_async(self.prepared.bind((0,)))) + break + except (OperationTimedOut, ReadTimeout): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb else: routing_key = struct.pack('>i', 0) for i in range(count): ss = SimpleStatement('SELECT * FROM %s.cf WHERE k = 0' % keyspace, consistency_level=consistency_level, routing_key=routing_key) - self.coordinator_stats.add_coordinator(session.execute_async(ss)) + while True: + try: + self.coordinator_stats.add_coordinator(session.execute_async(ss)) + break + except (OperationTimedOut, ReadTimeout): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb def test_roundrobin(self): use_singledc() diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index f5422ef233..bcdc24ac0a 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -135,12 +135,10 @@ def wait_for_up(cluster, node, wait=True): host = cluster.metadata.get_host(IP_FORMAT % node) time.sleep(0.1) if host and host.is_up: - # BUG: shouldn't have to, but we do - if wait: - log.debug("Sleeping 30s until host is up") - time.sleep(30) log.debug("Done waiting for node %s to be up", node) return + else: + log.debug("Host is still marked down, waiting") def wait_for_down(cluster, node, wait=True): @@ -149,10 +147,6 @@ def wait_for_down(cluster, node, wait=True): host = cluster.metadata.get_host(IP_FORMAT % node) time.sleep(0.1) if not host or not host.is_up: - # BUG: shouldn't have to, but we do - if wait: - log.debug("Sleeping 10s until host is down") - time.sleep(10) log.debug("Done waiting for node %s to be down", node) return else: From 290334be53dd660083878871c97ad313915969ee Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Mon, 20 Apr 2015 17:05:51 -0700 Subject: [PATCH 0044/2431] typo fix to enable TestCodeCoverage --- tests/integration/standard/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 0afc15b559..0e218cff09 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -362,7 +362,7 @@ def test_export_keyspace_schema_udts(self): "Protocol 3.0+ is required for UDT change events, currently testing against %r" % (PROTOCOL_VERSION,)) - if sys.version_info[2:] != (2, 7): + if sys.version_info[0:2] != (2, 7): raise unittest.SkipTest('This test compares static strings generated from dict items, which may change orders. Test with 2.7.') cluster = Cluster(protocol_version=PROTOCOL_VERSION) @@ -559,7 +559,7 @@ def test_legacy_tables(self): if get_server_versions()[0] < (2, 1, 0): raise unittest.SkipTest('Test schema output assumes 2.1.0+ options') - if sys.version_info[2:] != (2, 7): + if sys.version_info[0:2] != (2, 7): raise unittest.SkipTest('This test compares static strings generated from dict items, which may change orders. Test with 2.7.') cli_script = """CREATE KEYSPACE legacy From b0f130b1c903cb600afd09ff994f9a9db4f01a4d Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Mon, 20 Apr 2015 17:24:24 -0700 Subject: [PATCH 0045/2431] add twisted as a test dependency --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index 8fc8dad04a..a90f1ad5c5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,3 +9,4 @@ PyYAML pytz sure pure-sasl +twisted From 86436847aadabd3efd09d863ff76df8438876a83 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Mon, 20 Apr 2015 18:36:09 -0700 Subject: [PATCH 0046/2431] catch OperationTimedOut in consistency tests --- tests/integration/long/test_consistency.py | 13 +++++++++++-- tests/integration/long/utils.py | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/integration/long/test_consistency.py b/tests/integration/long/test_consistency.py index bf0cf7b937..db752cc4bd 100644 --- a/tests/integration/long/test_consistency.py +++ b/tests/integration/long/test_consistency.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import struct, logging, sys, traceback +import struct, logging, sys, traceback, time from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, Unavailable from cassandra.cluster import Cluster @@ -65,7 +65,15 @@ def _insert(self, session, keyspace, count, consistency_level=ConsistencyLevel.O for i in range(count): ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', consistency_level=consistency_level) - session.execute(ss) + while True: + try: + session.execute(ss) + break + except (OperationTimedOut, WriteTimeout): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + time.sleep(1) def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ONE): routing_key = struct.pack('>i', 0) @@ -81,6 +89,7 @@ def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ON ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb + time.sleep(1) def _assert_writes_succeed(self, session, keyspace, consistency_levels): for cl in consistency_levels: diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index bcdc24ac0a..e0e1e530d7 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -133,21 +133,21 @@ def ring(node): def wait_for_up(cluster, node, wait=True): while True: host = cluster.metadata.get_host(IP_FORMAT % node) - time.sleep(0.1) if host and host.is_up: log.debug("Done waiting for node %s to be up", node) return else: log.debug("Host is still marked down, waiting") + time.sleep(1) def wait_for_down(cluster, node, wait=True): log.debug("Waiting for node %s to be down", node) while True: host = cluster.metadata.get_host(IP_FORMAT % node) - time.sleep(0.1) if not host or not host.is_up: log.debug("Done waiting for node %s to be down", node) return else: log.debug("Host is still marked up, waiting") + time.sleep(1) From 7b15fae5c80093ae479c3a04aaa8db2da3d45f1d Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Tue, 21 Apr 2015 09:46:52 +0800 Subject: [PATCH 0047/2431] CASSANDRA-7814: code review comments --- cassandra/metadata.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 9b01a18c8e..8a11a64596 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -122,9 +122,8 @@ def rebuild_schema(self, ks_results, type_results, cf_results, col_results, trig keyspace_col_rows = col_def_rows.get(keyspace_meta.name, {}) keyspace_trigger_rows = trigger_rows.get(keyspace_meta.name, {}) for table_row in cf_def_rows.get(keyspace_meta.name, []): - self._add_table_metadata_to_ks( - keyspace_meta, table_row, keyspace_col_rows, - keyspace_trigger_rows) + table_meta = self._build_table_metadata(keyspace_meta, table_row, keyspace_col_rows, keyspace_trigger_rows) + keyspace_meta._add_table_metadata(table_meta) for usertype_row in usertype_rows.get(keyspace_meta.name, []): usertype = self._build_usertype(keyspace_meta.name, usertype_row) @@ -185,12 +184,11 @@ def table_changed(self, keyspace, table, cf_results, col_results, triggers_resul # the table was removed table_meta = keyspace_meta.tables.pop(table, None) if table_meta: - self._clear_table_indexes_in_ks(keyspace_meta, table_meta.name) + keyspace_meta._clear_table_indexes(table_meta.name) else: assert len(cf_results) == 1 - self._add_table_metadata_to_ks( - keyspace_meta, cf_results[0], {table: col_results}, - {table: triggers_result}) + table_meta = self._build_table_metadata(keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) + keyspace_meta._add_table_metadata(ctable_meta) def _keyspace_added(self, ksname): if self.token_map: @@ -216,20 +214,6 @@ def _build_usertype(self, keyspace, usertype_row): return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], usertype_row['field_names'], type_classes) - def _add_table_metadata_to_ks(self, keyspace_metadata, row, col_rows, trigger_rows): - self._clear_table_indexes_in_ks(keyspace_metadata, row["columnfamily_name"]) - - table_metadata = self._build_table_metadata(keyspace_metadata, row, col_rows, trigger_rows) - keyspace_metadata.tables[table_metadata.name] = table_metadata - for index_name, index_metadata in table_metadata.indexes.iteritems(): - keyspace_metadata.indexes[index_name] = index_metadata - - def _clear_table_indexes_in_ks(self, keyspace_metadata, table_name): - if table_name in keyspace_metadata.tables: - table_meta = keyspace_metadata.tables[table_name] - for index_name in table_meta.indexes: - keyspace_metadata.indexes.pop(index_name, None) - def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) @@ -803,6 +787,18 @@ def resolve_user_types(self, key, types, user_type_strings): self.resolve_user_types(field_type.typename, types, user_type_strings) user_type_strings.append(user_type.as_cql_query(formatted=True)) + def _add_table_metadata(self, table_metadata): + self._clear_table_indexes(table_metadata.name) + + self.tables[table_metadata.name] = table_metadata + for index_name, index_metadata in table_metadata.indexes.iteritems(): + self.indexes[index_name] = index_metadata + + def _clear_table_indexes(self, table_name): + if table_name in self.tables: + table_meta = self.tables[table_name] + for index_name in table_meta.indexes: + self.indexes.pop(index_name, None) class UserType(object): """ From 86944d70868522803508343c52ebc45412e9dd07 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 21 Apr 2015 13:30:17 -0500 Subject: [PATCH 0048/2431] Updates to index metadata indexes. Make indexes follow table alter Make keyspace meta clear indexes on table drop Add tests for this feature PYTHON-241 --- cassandra/metadata.py | 15 ++-- tests/integration/standard/test_metadata.py | 97 ++++++++++++++++++++- 2 files changed, 103 insertions(+), 9 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 8a11a64596..dfc69e3138 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -158,6 +158,7 @@ def keyspace_changed(self, keyspace, ks_results): if old_keyspace_meta: keyspace_meta.tables = old_keyspace_meta.tables keyspace_meta.user_types = old_keyspace_meta.user_types + keyspace_meta.indexes = old_keyspace_meta.indexes if (keyspace_meta.replication_strategy != old_keyspace_meta.replication_strategy): self._keyspace_updated(keyspace) else: @@ -182,13 +183,11 @@ def table_changed(self, keyspace, table, cf_results, col_results, triggers_resul if not cf_results: # the table was removed - table_meta = keyspace_meta.tables.pop(table, None) - if table_meta: - keyspace_meta._clear_table_indexes(table_meta.name) + keyspace_meta._drop_table_metadata(table) else: assert len(cf_results) == 1 table_meta = self._build_table_metadata(keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) - keyspace_meta._add_table_metadata(ctable_meta) + keyspace_meta._add_table_metadata(table_meta) def _keyspace_added(self, ksname): if self.token_map: @@ -788,15 +787,15 @@ def resolve_user_types(self, key, types, user_type_strings): user_type_strings.append(user_type.as_cql_query(formatted=True)) def _add_table_metadata(self, table_metadata): - self._clear_table_indexes(table_metadata.name) + self._drop_table_metadata(table_metadata.name) self.tables[table_metadata.name] = table_metadata for index_name, index_metadata in table_metadata.indexes.iteritems(): self.indexes[index_name] = index_metadata - def _clear_table_indexes(self, table_name): - if table_name in self.tables: - table_meta = self.tables[table_name] + def _drop_table_metadata(self, table_name): + table_meta = self.tables.pop(table_name, None) + if table_meta: for index_name in table_meta.indexes: self.indexes.pop(index_name, None) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index e5b2eab3ce..a3f1db7174 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -25,7 +25,7 @@ from cassandra import AlreadyExists from cassandra.cluster import Cluster -from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, +from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, IndexMetadata, Token, MD5Token, TokenMap, murmur3) from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host @@ -928,3 +928,98 @@ def test_keyspace_alter(self): new_keyspace_meta = self.cluster.metadata.keyspaces[name] self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) self.assertEqual(new_keyspace_meta.durable_writes, False) + + +class IndexMapTests(unittest.TestCase): + + keyspace_name = 'index_map_tests' + + @property + def table_name(self): + return self._testMethodName.lower() + + @classmethod + def setup_class(cls): + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.session = cls.cluster.connect() + try: + if cls.keyspace_name in cls.cluster.metadata.keyspaces: + cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) + + cls.session.execute( + """ + CREATE KEYSPACE %s + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}; + """ % cls.keyspace_name) + cls.session.set_keyspace(cls.keyspace_name) + except Exception: + cls.cluster.shutdown() + raise + + @classmethod + def teardown_class(cls): + try: + cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) + finally: + cls.cluster.shutdown() + + def create_basic_table(self): + self.session.execute("CREATE TABLE %s (k int PRIMARY KEY, a int)" % self.table_name) + + def drop_basic_table(self): + self.session.execute("DROP TABLE %s" % self.table_name) + + def test_index_updates(self): + self.create_basic_table() + + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertNotIn('a_idx', ks_meta.indexes) + self.assertNotIn('b_idx', ks_meta.indexes) + self.assertNotIn('a_idx', table_meta.indexes) + self.assertNotIn('b_idx', table_meta.indexes) + + self.session.execute("CREATE INDEX a_idx ON %s (a)" % self.table_name) + self.session.execute("ALTER TABLE %s ADD b int" % self.table_name) + self.session.execute("CREATE INDEX b_idx ON %s (b)" % self.table_name) + + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertIsInstance(ks_meta.indexes['a_idx'], IndexMetadata) + self.assertIsInstance(ks_meta.indexes['b_idx'], IndexMetadata) + self.assertIsInstance(table_meta.indexes['a_idx'], IndexMetadata) + self.assertIsInstance(table_meta.indexes['b_idx'], IndexMetadata) + + # both indexes updated when index dropped + self.session.execute("DROP INDEX a_idx") + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertNotIn('a_idx', ks_meta.indexes) + self.assertIsInstance(ks_meta.indexes['b_idx'], IndexMetadata) + self.assertNotIn('a_idx', table_meta.indexes) + self.assertIsInstance(table_meta.indexes['b_idx'], IndexMetadata) + + # keyspace index updated when table dropped + self.drop_basic_table() + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.assertNotIn(self.table_name, ks_meta.tables) + self.assertNotIn('a_idx', ks_meta.indexes) + self.assertNotIn('b_idx', ks_meta.indexes) + + def test_index_follows_alter(self): + self.create_basic_table() + + idx = self.table_name + '_idx' + self.session.execute("CREATE INDEX %s ON %s (a)" % (idx, self.table_name)) + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertIsInstance(ks_meta.indexes[idx], IndexMetadata) + self.assertIsInstance(table_meta.indexes[idx], IndexMetadata) + self.session.execute('ALTER KEYSPACE %s WITH durable_writes = false' % self.keyspace_name) + old_meta = ks_meta + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.assertIsNot(ks_meta, old_meta) + table_meta = ks_meta.tables[self.table_name] + self.assertIsInstance(ks_meta.indexes[idx], IndexMetadata) + self.assertIsInstance(table_meta.indexes[idx], IndexMetadata) + self.drop_basic_table() From b94e31be846c8366b3ab58dc8cb838fa5448db10 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 21 Apr 2015 14:55:37 -0500 Subject: [PATCH 0049/2431] Add connect_timeout to cluster and Connection.factory PYTHON-206 --- cassandra/cluster.py | 16 +++++++++++++--- cassandra/connection.py | 7 +++---- tests/integration/standard/test_connection.py | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 9aa5912b64..7183df8903 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -427,6 +427,14 @@ def auth_provider(self, value): See :attr:`.schema_event_refresh_window` for discussion of rationale """ + connect_timeout = 5 + """ + Timeout, in seconds, for creating new connections. + + This timeout covers the entire connection negotiation, including TCP + establishment, options passing, and authentication. + """ + sessions = None control_connection = None scheduler = None @@ -465,7 +473,8 @@ def __init__(self, control_connection_timeout=2.0, idle_heartbeat_interval=30, schema_event_refresh_window=2, - topology_event_refresh_window=10): + topology_event_refresh_window=10, + connect_timeout=5): """ Any of the mutable Cluster attributes may be set as keyword arguments to the constructor. @@ -518,6 +527,7 @@ def __init__(self, self.idle_heartbeat_interval = idle_heartbeat_interval self.schema_event_refresh_window = schema_event_refresh_window self.topology_event_refresh_window = topology_event_refresh_window + self.connect_timeout = connect_timeout self._listeners = set() self._listener_lock = Lock() @@ -707,11 +717,11 @@ def connection_factory(self, address, *args, **kwargs): Intended for internal use only. """ kwargs = self._make_connection_kwargs(address, kwargs) - return self.connection_class.factory(address, *args, **kwargs) + return self.connection_class.factory(address, self.connect_timeout, *args, **kwargs) def _make_connection_factory(self, host, *args, **kwargs): kwargs = self._make_connection_kwargs(host.address, kwargs) - return partial(self.connection_class.factory, host.address, *args, **kwargs) + return partial(self.connection_class.factory, host.address, self.connect_timeout, *args, **kwargs) def _make_connection_kwargs(self, address, kwargs_dict): if self._auth_provider_callable: diff --git a/cassandra/connection.py b/cassandra/connection.py index 6cbd434650..c239313c45 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -240,20 +240,19 @@ def handle_fork(self): pass @classmethod - def factory(cls, *args, **kwargs): + def factory(cls, host, timeout, *args, **kwargs): """ A factory function which returns connections which have succeeded in connecting and are ready for service (or raises an exception otherwise). """ - timeout = kwargs.pop('timeout', 5.0) - conn = cls(*args, **kwargs) + conn = cls(host, *args, **kwargs) conn.connected_event.wait(timeout) if conn.last_error: raise conn.last_error elif not conn.connected_event.is_set(): conn.close() - raise OperationTimedOut("Timed out creating connection") + raise OperationTimedOut("Timed out creating connection (%s seconds)" % timeout) else: return conn diff --git a/tests/integration/standard/test_connection.py b/tests/integration/standard/test_connection.py index e52280642b..3261adc346 100644 --- a/tests/integration/standard/test_connection.py +++ b/tests/integration/standard/test_connection.py @@ -58,7 +58,7 @@ def get_connection(self): e = None for i in range(5): try: - conn = self.klass.factory(protocol_version=PROTOCOL_VERSION) + conn = self.klass.factory(host='127.0.0.1', timeout=5, protocol_version=PROTOCOL_VERSION) break except (OperationTimedOut, NoHostAvailable) as e: continue From a84d4c0796da4550aa37c006cf1a66efcf62c428 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 22 Apr 2015 13:28:25 -0500 Subject: [PATCH 0050/2431] six.iteritems for newly added index dict --- cassandra/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index dfc69e3138..a3f2bcc3b4 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -790,7 +790,7 @@ def _add_table_metadata(self, table_metadata): self._drop_table_metadata(table_metadata.name) self.tables[table_metadata.name] = table_metadata - for index_name, index_metadata in table_metadata.indexes.iteritems(): + for index_name, index_metadata in six.iteritems(table_metadata.indexes): self.indexes[index_name] = index_metadata def _drop_table_metadata(self, table_name): From d0ee3bc5470aea7fa67d5b80189e51cb68ec64b6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 22 Apr 2015 12:58:50 -0500 Subject: [PATCH 0051/2431] Make DCAware LBP tolerate DC changes during query plan Fixes a RuntimeError that would raise if the DCAwareRoundRobinPolicy DC:host map changed during generation. PYTHON-297 --- cassandra/policies.py | 10 +-- tests/unit/test_policies.py | 154 ++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 5 deletions(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index 0dc36af76d..244df2411d 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -267,11 +267,11 @@ def make_query_plan(self, working_keyspace=None, query=None): for host in islice(cycle(local_live), pos, pos + len(local_live)): yield host - for dc, current_dc_hosts in six.iteritems(self._dc_live_hosts): - if dc == self.local_dc: - continue - - for host in current_dc_hosts[:self.used_hosts_per_remote_dc]: + # the dict can change, so get candidate DCs iterating over keys of a copy + other_dcs = [dc for dc in self._dc_live_hosts.copy().keys() if dc != self.local_dc] + for dc in other_dcs: + remote_live = self._dc_live_hosts.get(dc, ()) + for host in remote_live[:self.used_hosts_per_remote_dc]: yield host def on_up(self, host): diff --git a/tests/unit/test_policies.py b/tests/unit/test_policies.py index c6f049e09d..3e24f71fac 100644 --- a/tests/unit/test_policies.py +++ b/tests/unit/test_policies.py @@ -292,6 +292,160 @@ def test_status_updates(self): qplan = list(policy.make_query_plan()) self.assertEqual(qplan, []) + def test_modification_during_generation(self): + hosts = [Host(i, SimpleConvictionPolicy) for i in range(4)] + for h in hosts[:2]: + h.set_location_info("dc1", "rack1") + for h in hosts[2:]: + h.set_location_info("dc2", "rack1") + + policy = DCAwareRoundRobinPolicy("dc1", used_hosts_per_remote_dc=3) + policy.populate(Mock(), hosts) + + # The general concept here is to change thee internal state of the + # policy during plan generation. In this case we use a grey-box + # approach that changes specific things during known phases of the + # generator. + + new_host = Host(4, SimpleConvictionPolicy) + new_host.set_location_info("dc1", "rack1") + + # new local before iteration + plan = policy.make_query_plan() + policy.on_up(new_host) + # local list is not bound yet, so we get to see that one + self.assertEqual(len(list(plan)), 3 + 2) + + # remove local before iteration + plan = policy.make_query_plan() + policy.on_down(new_host) + # local list is not bound yet, so we don't see it + self.assertEqual(len(list(plan)), 2 + 2) + + # new local after starting iteration + plan = policy.make_query_plan() + next(plan) + policy.on_up(new_host) + # local list was is bound, and one consumed, so we only see the other original + self.assertEqual(len(list(plan)), 1 + 2) + + # remove local after traversing available + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + policy.on_down(new_host) + # we should be past the local list + self.assertEqual(len(list(plan)), 0 + 2) + + # REMOTES CHANGE + new_host.set_location_info("dc2", "rack1") + + # new remote after traversing local, but not starting remote + plan = policy.make_query_plan() + for _ in range(2): + next(plan) + policy.on_up(new_host) + # list is updated before we get to it + self.assertEqual(len(list(plan)), 0 + 3) + + # remove remote after traversing local, but not starting remote + plan = policy.make_query_plan() + for _ in range(2): + next(plan) + policy.on_down(new_host) + # list is updated before we get to it + self.assertEqual(len(list(plan)), 0 + 2) + + # new remote after traversing local, and starting remote + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + policy.on_up(new_host) + # slice is already made, and we've consumed one + self.assertEqual(len(list(plan)), 0 + 1) + + # remove remote after traversing local, and starting remote + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + policy.on_down(new_host) + # slice is created with all present, and we've consumed one + self.assertEqual(len(list(plan)), 0 + 2) + + # local DC disappears after finishing it, but not starting remote + plan = policy.make_query_plan() + for _ in range(2): + next(plan) + policy.on_down(hosts[0]) + policy.on_down(hosts[1]) + # dict traversal starts as normal + self.assertEqual(len(list(plan)), 0 + 2) + policy.on_up(hosts[0]) + policy.on_up(hosts[1]) + + # PYTHON-297 addresses the following cases, where DCs come and go + # during generation + # local DC disappears after finishing it, and starting remote + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + policy.on_down(hosts[0]) + policy.on_down(hosts[1]) + # dict traversal has begun and consumed one + self.assertEqual(len(list(plan)), 0 + 1) + policy.on_up(hosts[0]) + policy.on_up(hosts[1]) + + # remote DC disappears after finishing local, but not starting remote + plan = policy.make_query_plan() + for _ in range(2): + next(plan) + policy.on_down(hosts[2]) + policy.on_down(hosts[3]) + # nothing left + self.assertEqual(len(list(plan)), 0 + 0) + policy.on_up(hosts[2]) + policy.on_up(hosts[3]) + + # remote DC disappears while traversing it + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + policy.on_down(hosts[2]) + policy.on_down(hosts[3]) + # we continue with remainder of original list + self.assertEqual(len(list(plan)), 0 + 1) + policy.on_up(hosts[2]) + policy.on_up(hosts[3]) + + + another_host = Host(5, SimpleConvictionPolicy) + another_host.set_location_info("dc3", "rack1") + new_host.set_location_info("dc3", "rack1") + + # new DC while traversing remote + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + policy.on_up(new_host) + policy.on_up(another_host) + # we continue with remainder of original list + self.assertEqual(len(list(plan)), 0 + 1) + + # remote DC disappears after finishing it + plan = policy.make_query_plan() + for _ in range(3): + next(plan) + last_host_in_this_dc = next(plan) + if last_host_in_this_dc in (new_host, another_host): + down_hosts = [new_host, another_host] + else: + down_hosts = hosts[2:] + for h in down_hosts: + policy.on_down(h) + # the last DC has two + self.assertEqual(len(list(plan)), 0 + 2) + def test_no_live_nodes(self): """ Ensure query plan for a downed cluster will execute without errors From 8fe0fa5f7fadd9dad2a545dde6da79e118882639 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 23 Apr 2015 10:03:18 -0500 Subject: [PATCH 0052/2431] Remove race when adding new host during node/token rebuild Fixes an issue where a race could cause None return from Cluster.add_host during node/token list rebuild. This would cause None to be inserted into the Token map, and subsequently blow up TokenAware LBP creating a query plan. PYTHON-298 --- cassandra/cluster.py | 28 +++++++++++---------- cassandra/metadata.py | 28 +++++++++------------ tests/integration/standard/test_metadata.py | 1 - 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index e523e38162..c7b2be7b73 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -64,14 +64,13 @@ from cassandra.policies import (RoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, RetryPolicy) -from cassandra.pool import (_ReconnectionHandler, _HostReconnectionHandler, +from cassandra.pool import (Host, _ReconnectionHandler, _HostReconnectionHandler, HostConnectionPool, HostConnection, NoConnectionsAvailable) from cassandra.query import (SimpleStatement, PreparedStatement, BoundStatement, BatchStatement, bind_params, QueryTrace, Statement, named_tuple_factory, dict_factory, FETCH_SIZE_UNSET) - def _is_eventlet_monkey_patched(): if 'eventlet.patcher' not in sys.modules: return False @@ -525,7 +524,7 @@ def __init__(self, # let Session objects be GC'ed (and shutdown) when the user no longer # holds a reference. self.sessions = WeakSet() - self.metadata = Metadata(self) + self.metadata = Metadata() self.control_connection = None self._prepared_statements = WeakValueDictionary() self._prepared_statement_lock = Lock() @@ -743,8 +742,8 @@ def connect(self, keyspace=None): self.connection_class.initialize_reactor() atexit.register(partial(_shutdown_cluster, self)) for address in self.contact_points: - host = self.add_host(address, signal=False) - if host: + host, new = self.add_host(address, signal=False) + if new: host.set_up() for listener in self.listeners: listener.on_add(host) @@ -1069,15 +1068,17 @@ def signal_connection_failure(self, host, connection_exc, is_host_addition, expe def add_host(self, address, datacenter=None, rack=None, signal=True, refresh_nodes=True): """ Called when adding initial contact points and when the control - connection subsequently discovers a new node. Intended for internal - use only. + connection subsequently discovers a new node. + Returns a Host instance, and a flag indicating whether it was new in + the metadata. + Intended for internal use only. """ - new_host = self.metadata.add_host(address, datacenter, rack) - if new_host and signal: - log.info("New Cassandra host %r discovered", new_host) - self.on_add(new_host, refresh_nodes) + host, new = self.metadata.add_or_return_host(Host(address, self.conviction_policy_factory, datacenter, rack)) + if new and signal: + log.info("New Cassandra host %r discovered", host) + self.on_add(host, refresh_nodes) - return new_host + return host, new def remove_host(self, host): """ @@ -2158,6 +2159,7 @@ def refresh_node_list_and_token_map(self, force_token_rebuild=False): def _refresh_node_list_and_token_map(self, connection, preloaded_results=None, force_token_rebuild=False): + if preloaded_results: log.debug("[control connection] Refreshing node list and token map using preloaded results") peers_result = preloaded_results[0] @@ -2215,7 +2217,7 @@ def _refresh_node_list_and_token_map(self, connection, preloaded_results=None, rack = row.get("rack") if host is None: log.debug("[control connection] Found new host to connect to: %s", addr) - host = self._cluster.add_host(addr, datacenter, rack, signal=True, refresh_nodes=False) + host, _ = self._cluster.add_host(addr, datacenter, rack, signal=True, refresh_nodes=False) should_rebuild_token_map = True else: should_rebuild_token_map |= self._update_location_info(host, datacenter, rack) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 1f88d8f9e6..8ca84fc846 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -20,7 +20,6 @@ import logging import re from threading import RLock -import weakref import six murmur3 = None @@ -31,7 +30,6 @@ import cassandra.cqltypes as types from cassandra.marshal import varint_unpack -from cassandra.pool import Host from cassandra.util import OrderedDict log = logging.getLogger(__name__) @@ -72,11 +70,7 @@ class Metadata(object): token_map = None """ A :class:`~.TokenMap` instance describing the ring topology. """ - def __init__(self, cluster): - # use a weak reference so that the Cluster object can be GC'ed. - # Normally the cycle detector would handle this, but implementing - # __del__ disables that. - self.cluster_ref = weakref.ref(cluster) + def __init__(self): self.keyspaces = {} self._hosts = {} self._hosts_lock = RLock() @@ -451,17 +445,19 @@ def can_support_partitioner(self): else: return True - def add_host(self, address, datacenter, rack): - cluster = self.cluster_ref() + def add_or_return_host(self, host): + """ + Returns a tuple (host, new), where ``host`` is a Host + instance, and ``new`` is a bool indicating whether + the host was newly added. + """ with self._hosts_lock: - if address not in self._hosts: - new_host = Host( - address, cluster.conviction_policy_factory, datacenter, rack) - self._hosts[address] = new_host - else: - return None + try: + return self._hosts[host.address], False + except KeyError: + self._hosts[host.address] = host + return host, True - return new_host def remove_host(self, host): with self._hosts_lock: diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index e5b2eab3ce..eab46d0dce 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -143,7 +143,6 @@ def test_basic_table_meta_properties(self): self.cluster.control_connection.refresh_schema() meta = self.cluster.metadata - self.assertNotEqual(meta.cluster_ref, None) self.assertNotEqual(meta.cluster_name, None) self.assertTrue(self.ksname in meta.keyspaces) ksmeta = meta.keyspaces[self.ksname] From 25f10a574c1cbb789eaeb19c16b30dfd92bb7d90 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 23 Apr 2015 10:47:34 -0500 Subject: [PATCH 0053/2431] Update unit test for new Metadata init signature. --- tests/unit/test_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 0c59598a23..5ce78f2e25 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -316,7 +316,7 @@ def test_build_index_as_cql(self): column_meta.name = 'column_name_here' column_meta.table.name = 'table_name_here' column_meta.table.keyspace.name = 'keyspace_name_here' - meta_model = Metadata(Mock()) + meta_model = Metadata() row = {'index_name': 'index_name_here', 'index_type': 'index_type_here'} index_meta = meta_model._build_index_metadata(column_meta, row) From c2b999c5ee1f6931f581d0e77517287f4ed7935d Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 23 Apr 2015 12:47:23 -0500 Subject: [PATCH 0054/2431] Ensure serial consistency gets set on statements and messages PYTHON-299 --- cassandra/query.py | 1 + tests/integration/standard/test_query.py | 44 ++++++++++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/cassandra/query.py b/cassandra/query.py index e973ed2b2e..8bc156f1bb 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -248,6 +248,7 @@ def _set_serial_consistency_level(self, serial_consistency_level): raise ValueError( "serial_consistency_level must be either ConsistencyLevel.SERIAL " "or ConsistencyLevel.LOCAL_SERIAL") + self._serial_consistency_level = serial_consistency_level def _del_serial_consistency_level(self): self._serial_consistency_level = None diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 780f307891..f207c45367 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -325,14 +325,22 @@ def test_conditional_update(self): statement = SimpleStatement( "UPDATE test3rf.test SET v=1 WHERE k=0 IF v=1", serial_consistency_level=ConsistencyLevel.SERIAL) - result = self.session.execute(statement) + # crazy test, but PYTHON-299 + # TODO: expand to check more parameters get passed to statement, and on to messages + self.assertEqual(statement.serial_consistency_level, ConsistencyLevel.SERIAL) + future = self.session.execute_async(statement) + result = future.result() + self.assertEqual(future.message.serial_consistency_level, ConsistencyLevel.SERIAL) self.assertEqual(1, len(result)) self.assertFalse(result[0].applied) statement = SimpleStatement( "UPDATE test3rf.test SET v=1 WHERE k=0 IF v=0", - serial_consistency_level=ConsistencyLevel.SERIAL) - result = self.session.execute(statement) + serial_consistency_level=ConsistencyLevel.LOCAL_SERIAL) + self.assertEqual(statement.serial_consistency_level, ConsistencyLevel.LOCAL_SERIAL) + future = self.session.execute_async(statement) + result = future.result() + self.assertEqual(future.message.serial_consistency_level, ConsistencyLevel.LOCAL_SERIAL) self.assertEqual(1, len(result)) self.assertTrue(result[0].applied) @@ -342,15 +350,39 @@ def test_conditional_update_with_prepared_statements(self): "UPDATE test3rf.test SET v=1 WHERE k=0 IF v=2") statement.serial_consistency_level = ConsistencyLevel.SERIAL - result = self.session.execute(statement) + future = self.session.execute_async(statement) + result = future.result() + self.assertEqual(future.message.serial_consistency_level, ConsistencyLevel.SERIAL) self.assertEqual(1, len(result)) self.assertFalse(result[0].applied) statement = self.session.prepare( "UPDATE test3rf.test SET v=1 WHERE k=0 IF v=0") bound = statement.bind(()) - bound.serial_consistency_level = ConsistencyLevel.SERIAL - result = self.session.execute(statement) + bound.serial_consistency_level = ConsistencyLevel.LOCAL_SERIAL + future = self.session.execute_async(bound) + result = future.result() + self.assertEqual(future.message.serial_consistency_level, ConsistencyLevel.LOCAL_SERIAL) + self.assertEqual(1, len(result)) + self.assertTrue(result[0].applied) + + def test_conditional_update_with_batch_statements(self): + self.session.execute("INSERT INTO test3rf.test (k, v) VALUES (0, 0)") + statement = BatchStatement(serial_consistency_level=ConsistencyLevel.SERIAL) + statement.add("UPDATE test3rf.test SET v=1 WHERE k=0 IF v=1") + self.assertEqual(statement.serial_consistency_level, ConsistencyLevel.SERIAL) + future = self.session.execute_async(statement) + result = future.result() + self.assertEqual(future.message.serial_consistency_level, ConsistencyLevel.SERIAL) + self.assertEqual(1, len(result)) + self.assertFalse(result[0].applied) + + statement = BatchStatement(serial_consistency_level=ConsistencyLevel.LOCAL_SERIAL) + statement.add("UPDATE test3rf.test SET v=1 WHERE k=0 IF v=0") + self.assertEqual(statement.serial_consistency_level, ConsistencyLevel.LOCAL_SERIAL) + future = self.session.execute_async(statement) + result = future.result() + self.assertEqual(future.message.serial_consistency_level, ConsistencyLevel.LOCAL_SERIAL) self.assertEqual(1, len(result)) self.assertTrue(result[0].applied) From 28c870daf60ecacd96c4855d8d43a4ede0a14832 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 23 Apr 2015 13:54:00 -0500 Subject: [PATCH 0055/2431] Release ver and changelog update for 2.5.1 Conflicts: cassandra/__init__.py --- CHANGELOG.rst | 10 ++++++++++ cassandra/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af2d611309..6000f16012 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,13 @@ +2.5.1 +===== +April 23, 2015 + +Bug Fixes +--------- +* Fix thread safety in DC-aware load balancing policy (PYTHON-297) +* Fix race condition in node/token rebuild (PYTHON-298) +* Set and send serial consistency parameter (PYTHON-299) + 2.5.0 ===== March 30, 2015 diff --git a/cassandra/__init__.py b/cassandra/__init__.py index fa26054fc2..2be5700099 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 5, 0) +__version_info__ = (2, 5, 1) __version__ = '.'.join(map(str, __version_info__)) From 931337b4a253d89a0ac309b305f38f6c8af0ce10 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 23 Apr 2015 18:06:39 -0700 Subject: [PATCH 0056/2431] Update date/time datatype tests to run on C* 3.0 As per CASSANDRA-7523, the new date and time types have been pushed back to C* 3.0 / protocol v4. --- tests/integration/datatype_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/datatype_utils.py b/tests/integration/datatype_utils.py index bd76c36fec..fb437d7539 100644 --- a/tests/integration/datatype_utils.py +++ b/tests/integration/datatype_utils.py @@ -57,7 +57,7 @@ def update_datatypes(): if _cass_version >= (2, 1, 0): COLLECTION_TYPES.append('tuple') - if _cass_version >= (2, 1, 5): + if _cass_version >= (3, 0, 0): PRIMITIVE_DATATYPES.append('date') PRIMITIVE_DATATYPES.append('time') From 9f0ca79a700ea06881ae67bd6a401adb31cef4df Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 24 Apr 2015 10:24:37 -0500 Subject: [PATCH 0057/2431] Add docs for ResponseFuture.custom_paylaod --- cassandra/cluster.py | 11 +++++++++++ docs/api/cassandra/cluster.rst | 2 ++ 2 files changed, 13 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 28885ae9aa..7d982e15d9 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2677,6 +2677,17 @@ def has_more_pages(self): @property def custom_payload(self): + """ + The custom payload returned from the server, if any. This will only be + set by Cassandra servers implementing a custom QueryHandler, and only + for protocol_version 4+. + + Ensure the future is complete before trying to access this property + (call :meth:`.result()`, or after callback is invoked). + Otherwise it may throw if the response has not been received. + + :return: :ref:`custom_payload`. + """ if not self._event.is_set(): raise Exception("custom_payload cannot be retrieved before ResponseFuture is finalized") return self._custom_payload diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 6532c9a026..e6e3cf7660 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -102,6 +102,8 @@ .. automethod:: get_query_trace() + .. autoattribute:: custom_payload() + .. autoattribute:: has_more_pages .. automethod:: start_fetching_next_page() From 3f2a51ac0d12304a38031543c2c6afff31e9d5f1 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 24 Apr 2015 13:41:50 -0700 Subject: [PATCH 0058/2431] handle Timeout errors in LargeDataTests --- tests/integration/long/test_large_data.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/integration/long/test_large_data.py b/tests/integration/long/test_large_data.py index 7edd28b48c..acb203568b 100644 --- a/tests/integration/long/test_large_data.py +++ b/tests/integration/long/test_large_data.py @@ -18,15 +18,22 @@ from queue import Queue, Empty # noqa from struct import pack -import unittest +import logging, sys, traceback, time -from cassandra import ConsistencyLevel +from cassandra import ConsistencyLevel, OperationTimedOut, WriteTimeout from cassandra.cluster import Cluster from cassandra.query import dict_factory from cassandra.query import SimpleStatement from tests.integration import use_singledc, PROTOCOL_VERSION from tests.integration.long.utils import create_schema +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + +log = logging.getLogger(__name__) + def setup_module(): use_singledc() @@ -71,6 +78,11 @@ def batch_futures(self, session, statement_generator): while True: try: futures.get_nowait().result() + except (OperationTimedOut, WriteTimeout): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + time.sleep(1) except Empty: break @@ -80,6 +92,11 @@ def batch_futures(self, session, statement_generator): while True: try: futures.get_nowait().result() + except (OperationTimedOut, WriteTimeout): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + time.sleep(1) except Empty: break From e67e215df51f74d2032090253daaf0c1158f1e9b Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 24 Apr 2015 14:50:03 -0700 Subject: [PATCH 0059/2431] Refactored SchemaTests --- tests/integration/long/test_schema.py | 125 +++++++++++++------------- 1 file changed, 61 insertions(+), 64 deletions(-) diff --git a/tests/integration/long/test_schema.py b/tests/integration/long/test_schema.py index af51924045..6f4cc0f1f4 100644 --- a/tests/integration/long/test_schema.py +++ b/tests/integration/long/test_schema.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging, sys, traceback from cassandra import ConsistencyLevel, OperationTimedOut from cassandra.cluster import Cluster +from cassandra.protocol import ConfigurationException from cassandra.query import SimpleStatement from tests.integration import use_singledc, PROTOCOL_VERSION @@ -43,91 +44,87 @@ def teardown_class(cls): cls.cluster.shutdown() def test_recreates(self): + """ + Basic test for repeated schema creation and use, using many different keyspaces + """ + session = self.session - replication_factor = 3 for i in range(2): - for keyspace in range(5): - keyspace = 'ks_%s' % keyspace - results = session.execute('SELECT keyspace_name FROM system.schema_keyspaces') + for keyspace_number in range(5): + keyspace = "ks_{0}".format(keyspace_number) + + results = session.execute("SELECT keyspace_name FROM system.schema_keyspaces") existing_keyspaces = [row[0] for row in results] if keyspace in existing_keyspaces: - ddl = 'DROP KEYSPACE %s' % keyspace - log.debug(ddl) - session.execute(ddl) - - ddl = """ - CREATE KEYSPACE %s - WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '%s'} - """ % (keyspace, str(replication_factor)) - log.debug(ddl) - session.execute(ddl) - - ddl = 'CREATE TABLE %s.cf (k int PRIMARY KEY, i int)' % keyspace - log.debug(ddl) - session.execute(ddl) - - statement = 'USE %s' % keyspace - log.debug(ddl) - session.execute(statement) - - statement = 'INSERT INTO %s(k, i) VALUES (0, 0)' % 'cf' - log.debug(statement) - ss = SimpleStatement(statement, - consistency_level=ConsistencyLevel.QUORUM) + drop = "DROP KEYSPACE {0}".format(keyspace) + log.debug(drop) + session.execute(drop) + + create = "CREATE KEYSPACE {0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 3}}".format(keyspace) + log.debug(create) + session.execute(create) + + create = "CREATE TABLE {0}.cf (k int PRIMARY KEY, i int)".format(keyspace) + log.debug(create) + session.execute(create) + + use = "USE {0}".format(keyspace) + log.debug(use) + session.execute(use) + + insert = "INSERT INTO cf (k, i) VALUES (0, 0)" + log.debug(insert) + ss = SimpleStatement(insert, consistency_level=ConsistencyLevel.QUORUM) session.execute(ss) def test_for_schema_disagreements_different_keyspaces(self): + """ + Tests for any schema disagreements using many different keyspaces + """ + session = self.session for i in xrange(30): try: - session.execute(''' - CREATE KEYSPACE test_%s - WITH replication = {'class': 'SimpleStrategy', - 'replication_factor': 1} - ''' % i) - - session.execute(''' - CREATE TABLE test_%s.cf ( - key int, - value int, - PRIMARY KEY (key)) - ''' % i) + session.execute("CREATE KEYSPACE test_{0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}".format(i)) + session.execute("CREATE TABLE test_{0}.cf (key int PRIMARY KEY, value int)".format(i)) for j in xrange(100): - session.execute('INSERT INTO test_%s.cf (key, value) VALUES (%s, %s)' % (i, j, j)) - - session.execute(''' - DROP KEYSPACE test_%s - ''' % i) + session.execute("INSERT INTO test_{0}.cf (key, value) VALUES ({1}, {1})".format(i, j)) except OperationTimedOut: - pass + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + finally: + try: + session.execute("DROP KEYSPACE test_{0}".format(i)) + except ConfigurationException: + # We're good, the keyspace was never created due to OperationTimedOut + pass def test_for_schema_disagreements_same_keyspace(self): + """ + Tests for any schema disagreements using the same keyspace multiple times + """ + cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() for i in xrange(30): try: - session.execute(''' - CREATE KEYSPACE test - WITH replication = {'class': 'SimpleStrategy', - 'replication_factor': 1} - ''') - - session.execute(''' - CREATE TABLE test.cf ( - key int, - value int, - PRIMARY KEY (key)) - ''') + session.execute("CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}") + session.execute("CREATE TABLE test.cf (key int PRIMARY KEY, value int)") for j in xrange(100): - session.execute('INSERT INTO test.cf (key, value) VALUES (%s, %s)' % (j, j)) - - session.execute(''' - DROP KEYSPACE test - ''') + session.execute("INSERT INTO test.cf (key, value) VALUES ({0}, {0})".format(j)) except OperationTimedOut: - pass + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + finally: + try: + session.execute("DROP KEYSPACE test") + except ConfigurationException: + # We're good, the keyspace was never created due to OperationTimedOut + pass From 2d8ee3cae0486da39749cb6c93b83150db5e42cc Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 24 Apr 2015 15:11:54 -0700 Subject: [PATCH 0060/2431] Handle ReadTimeout in LightweightTransactionTests --- tests/integration/standard/test_query.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index f207c45367..b30c387b01 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -442,15 +442,16 @@ def test_no_connection_refused_on_timeout(self): for (success, result) in results: if success: continue - # In this case result is an exception - if type(result).__name__ == "NoHostAvailable": - self.fail("PYTHON-91: Disconnected from Cassandra: %s" % result.message) - break - if type(result).__name__ == "WriteTimeout": - received_timeout = True - continue - self.fail("Unexpected exception %s: %s" % (type(result).__name__, result.message)) - break + else: + # In this case result is an exception + if type(result).__name__ == "NoHostAvailable": + self.fail("PYTHON-91: Disconnected from Cassandra: %s" % result.message) + if type(result).__name__ == "WriteTimeout": + received_timeout = True + continue + if type(result).__name__ == "ReadTimeout": + continue + self.fail("Unexpected exception %s: %s" % (type(result).__name__, result.message)) # Make sure test passed self.assertTrue(received_timeout) From 6423e938a45f1261e8fc8b7431c0efdce188cbe9 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 24 Apr 2015 19:16:46 -0700 Subject: [PATCH 0061/2431] handle OperationTimedOut in SchemaTests --- tests/integration/long/test_schema.py | 32 ++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/integration/long/test_schema.py b/tests/integration/long/test_schema.py index 6f4cc0f1f4..806726cf32 100644 --- a/tests/integration/long/test_schema.py +++ b/tests/integration/long/test_schema.py @@ -97,11 +97,17 @@ def test_for_schema_disagreements_different_keyspaces(self): log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb finally: - try: - session.execute("DROP KEYSPACE test_{0}".format(i)) - except ConfigurationException: - # We're good, the keyspace was never created due to OperationTimedOut - pass + while True: + try: + session.execute("DROP KEYSPACE test_{0}".format(i)) + break + except OperationTimedOut: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + except ConfigurationException: + # We're good, the keyspace was never created due to OperationTimedOut + break def test_for_schema_disagreements_same_keyspace(self): """ @@ -123,8 +129,14 @@ def test_for_schema_disagreements_same_keyspace(self): log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb finally: - try: - session.execute("DROP KEYSPACE test") - except ConfigurationException: - # We're good, the keyspace was never created due to OperationTimedOut - pass + while True: + try: + session.execute("DROP KEYSPACE test") + break + except OperationTimedOut: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + except ConfigurationException: + # We're good, the keyspace was never created due to OperationTimedOut + break From 1cffc9168e336e7d27d402b8abd34dca28fbd377 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 24 Apr 2015 20:10:19 -0700 Subject: [PATCH 0062/2431] Handle OperationTimedOut in SchemaMetadataTests in TearDown --- tests/integration/standard/test_metadata.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 9b5e73c6f9..6e2bfc402c 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -19,10 +19,9 @@ import difflib from mock import Mock -import six -import sys +import six, logging, sys, traceback -from cassandra import AlreadyExists +from cassandra import AlreadyExists, OperationTimedOut from cassandra.cluster import Cluster from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, IndexMetadata, @@ -33,6 +32,8 @@ from tests.integration import (get_cluster, use_singledc, PROTOCOL_VERSION, get_server_versions) +log = logging.getLogger(__name__) + def setup_module(): use_singledc() @@ -54,8 +55,15 @@ def setUp(self): self.session.execute("CREATE KEYSPACE schemametadatatest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}") def tearDown(self): - self.session.execute("DROP KEYSPACE schemametadatatest") - self.cluster.shutdown() + while True: + try: + self.session.execute("DROP KEYSPACE schemametadatatest") + self.cluster.shutdown() + break + except OperationTimedOut: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb def make_create_statement(self, partition_cols, clustering_cols=None, other_cols=None, compact=False): clustering_cols = clustering_cols or [] From 7c03eacf70b3b203fe4d819a3e62e8b7a82bbe5e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 27 Apr 2015 09:37:31 -0500 Subject: [PATCH 0063/2431] Fix cqlengine UDT example in docs --- docs/cqlengine/models.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/cqlengine/models.rst b/docs/cqlengine/models.rst index 002ce3de65..dffd06fb3f 100644 --- a/docs/cqlengine/models.rst +++ b/docs/cqlengine/models.rst @@ -199,12 +199,10 @@ are only created, presisted, and queried via table Models. A short example to in name = Text(primary_key=True) addr = UserDefinedType(address) - sync_table(users) - - users.create(name="Joe", addr=address(street="Easy St.", zip=99999)) + users.create(name="Joe", addr=address(street="Easy St.", zipcode=99999)) user = users.objects(name="Joe")[0] print user.name, user.addr - # Joe {'street': Easy St., 'zipcode': None} + # Joe address(street=u'Easy St.', zipcode=99999) UDTs are modeled by inheriting :class:`~.usertype.UserType`, and setting column type attributes. Types are then used in defining models by declaring a column of type :class:`~.columns.UserDefinedType`, with the ``UserType`` class as a parameter. From d3d60206d49fc0ffde71daa1b75b8e1f07d1ade9 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 10:42:36 -0500 Subject: [PATCH 0064/2431] Basic timer management, and timer impl for Asyncore --- cassandra/connection.py | 63 +++++++++++++++++++++++++++++++++ cassandra/io/asyncorereactor.py | 49 ++++++++++++------------- 2 files changed, 85 insertions(+), 27 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index c239313c45..f5ca774a0d 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -29,6 +29,7 @@ from six.moves.queue import Queue, Empty # noqa import six +from six.moves import queue from six.moves import range from cassandra import ConsistencyLevel, AuthenticationFailed, OperationTimedOut @@ -256,6 +257,10 @@ def factory(cls, host, timeout, *args, **kwargs): else: return conn + @classmethod + def create_timer(cls, timeout, callback): + raise NotImplementedError() + def close(self): raise NotImplementedError() @@ -871,3 +876,61 @@ def stop(self): def _raise_if_stopped(self): if self._shutdown_event.is_set(): raise self.ShutdownException() + + +class Timer(object): + + cancelled = False + + def __init__(self, timeout, callback): + self.end = time.time() + timeout + self.callback = callback + if timeout < 0: + self.on_timeout() + + def __lt__(self, other): + return self.end < other.end + + def cancel(self): + self.callback = self._noop + self.cancelled = True + + def on_timeout(self): + self.callback() + + def _noop(self): + pass + + +class TimerManager(object): + + def __init__(self): + self._timers = queue.PriorityQueue() + + def add_timer(self, timer): + self._timers.put_nowait(timer) + + def service_timeouts(self): + now = time.time() + while not self._timers.empty(): + timer = self._timers.get_nowait() + if timer.end < now: + timer.on_timeout() + else: + self._timers.put_nowait(timer) + break + + @property + def next_timeout(self): + try: + return self._timers.queue[0].end + except IndexError: + pass + + @property + def next_offset(self): + try: + next_end = self._timers.queue[0].end + return next_end - time.time() + except IndexError: + pass diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index ef687c388c..eeb319bb65 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -19,11 +19,12 @@ import socket import sys from threading import Event, Lock, Thread +import time import weakref from six.moves import range -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN, errorcode +from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN try: from weakref import WeakSet except ImportError: @@ -36,9 +37,9 @@ except ImportError: ssl = None # NOQA -from cassandra import OperationTimedOut from cassandra.connection import (Connection, ConnectionShutdown, - ConnectionException, NONBLOCKING) + ConnectionException, NONBLOCKING, + Timer, TimerManager) from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -55,15 +56,17 @@ def _cleanup(loop_weakref): class AsyncoreLoop(object): + def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() self._started = False self._shutdown = False - self._conns_lock = Lock() - self._conns = WeakSet() self._thread = None + + self._timers = TimerManager() + atexit.register(partial(_cleanup, weakref.ref(self))) def maybe_start(self): @@ -86,24 +89,22 @@ def maybe_start(self): def _run_loop(self): log.debug("Starting asyncore event loop") with self._loop_lock: - while True: + while not self._shutdown: try: - asyncore.loop(timeout=0.001, use_poll=True, count=1000) + asyncore.loop(timeout=0.001, use_poll=True, count=100) + self._timers.service_timeouts() + if not asyncore.socket_map: + time.sleep(0.005) except Exception: log.debug("Asyncore event loop stopped unexepectedly", exc_info=True) break - - if self._shutdown: - break - - with self._conns_lock: - if len(self._conns) == 0: - break - self._started = False log.debug("Asyncore event loop ended") + def add_timer(self, timer): + self._timers.add_timer(timer) + def _cleanup(self): self._shutdown = True if not self._thread: @@ -118,14 +119,6 @@ def _cleanup(self): log.debug("Event loop thread was joined") - def connection_created(self, connection): - with self._conns_lock: - self._conns.add(connection) - - def connection_destroyed(self, connection): - with self._conns_lock: - self._conns.discard(connection) - class AsyncoreConnection(Connection, asyncore.dispatcher): """ @@ -156,6 +149,12 @@ def handle_fork(cls): cls._loop._cleanup() cls._loop = None + @classmethod + def create_timer(self, timeout, callback): + timer = Timer(timeout, callback) + self._loop.add_timer(timer) + return timer + def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) asyncore.dispatcher.__init__(self) @@ -166,8 +165,6 @@ def __init__(self, *args, **kwargs): self.deque = deque() self.deque_lock = Lock() - self._loop.connection_created(self) - sockerr = None addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) for (af, socktype, proto, canonname, sockaddr) in addresses: @@ -240,8 +237,6 @@ def close(self): asyncore.dispatcher.close(self) log.debug("Closed socket to %s", self.host) - self._loop.connection_destroyed(self) - if not self.is_defunct: self.error_all_callbacks( ConnectionShutdown("Connection to %s was closed" % self.host)) From 4a9e08ed7a12c0bd59c7aa96292c8c69a1a163ac Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 10:43:55 -0500 Subject: [PATCH 0065/2431] Timer for libevreactor --- cassandra/io/libevreactor.py | 44 +++++++++--- cassandra/io/libevwrapper.c | 131 +++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 9 deletions(-) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 93b4c97854..174cfd859c 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -20,10 +20,10 @@ from threading import Event, Lock, Thread import weakref -from six.moves import xrange +from six.moves import range -from cassandra import OperationTimedOut -from cassandra.connection import Connection, ConnectionShutdown, NONBLOCKING +from cassandra.connection import (Connection, ConnectionShutdown, + NONBLOCKING, Timer, TimerManager) from cassandra.protocol import RegisterMessage try: import cassandra.io.libevwrapper as libev @@ -40,7 +40,7 @@ try: import ssl except ImportError: - ssl = None # NOQA + ssl = None # NOQA log = logging.getLogger(__name__) @@ -50,7 +50,6 @@ def _cleanup(loop_weakref): loop = loop_weakref() except ReferenceError: return - loop._cleanup() @@ -85,10 +84,10 @@ def __init__(self): self._loop.unref() self._preparer.start() - atexit.register(partial(_cleanup, weakref.ref(self))) + self._timers = TimerManager() + self._loop_timer = libev.Timer(self._loop, self._on_loop_timer) - def notify(self): - self._notifier.send() + atexit.register(partial(_cleanup, weakref.ref(self))) def maybe_start(self): should_start = False @@ -133,6 +132,7 @@ def _cleanup(self): conn._read_watcher.stop() del conn._read_watcher + self.notify() # wake the timer watcher log.debug("Waiting for event loop thread to join...") self._thread.join(timeout=1.0) if self._thread.is_alive(): @@ -143,6 +143,23 @@ def _cleanup(self): log.debug("Event loop thread was joined") self._loop = None + def add_timer(self, timer): + self._timers.add_timer(timer) + self._notifier.send() # wake up in case this timer is earlier + + def _update_timer(self): + if not self._shutdown: + offset = self._timers.next_offset or 100000 # none pending; will be updated again when something new happens + self._loop_timer.start(offset) + else: + self._loop_timer.stop() + + def _on_loop_timer(self): + self._timers.service_timeouts() + + def notify(self): + self._notifier.send() + def connection_created(self, conn): with self._conn_set_lock: new_live_conns = self._live_conns.copy() @@ -205,6 +222,9 @@ def _loop_will_run(self, prepare): changed = True + # TODO: update to do connection management, timer updates through dedicaterd async 'notifier' callbacks + self._update_timer() + if changed: self._notifier.send() @@ -236,6 +256,12 @@ def handle_fork(cls): cls._libevloop._cleanup() cls._libevloop = None + @classmethod + def create_timer(self, timeout, callback): + timer = Timer(timeout, callback) + self._libevloop.add_timer(timer) + return timer + def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) @@ -361,7 +387,7 @@ def push(self, data): sabs = self.out_buffer_size if len(data) > sabs: chunks = [] - for i in xrange(0, len(data), sabs): + for i in range(0, len(data), sabs): chunks.append(data[i:i + sabs]) else: chunks = [data] diff --git a/cassandra/io/libevwrapper.c b/cassandra/io/libevwrapper.c index cbac83b277..1c52a96464 100644 --- a/cassandra/io/libevwrapper.c +++ b/cassandra/io/libevwrapper.c @@ -451,6 +451,129 @@ static PyTypeObject libevwrapper_PrepareType = { (initproc)Prepare_init, /* tp_init */ }; +typedef struct libevwrapper_Timer { + PyObject_HEAD + struct ev_timer timer; + struct libevwrapper_Loop *loop; + PyObject *callback; +} libevwrapper_Timer; + +static void +Timer_dealloc(libevwrapper_Timer *self) { + Py_XDECREF(self->loop); + Py_XDECREF(self->callback); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static void timer_callback(struct ev_loop *loop, ev_timer *watcher, int revents) { + libevwrapper_Timer *self = watcher->data; + + PyObject *result = NULL; + PyGILState_STATE gstate; + + gstate = PyGILState_Ensure(); + result = PyObject_CallFunction(self->callback, NULL); + if (!result) { + PyErr_WriteUnraisable(self->callback); + } + Py_XDECREF(result); + + PyGILState_Release(gstate); +} + +static int +Timer_init(libevwrapper_Timer *self, PyObject *args, PyObject *kwds) { + PyObject *callback; + PyObject *loop; + + if (!PyArg_ParseTuple(args, "OO", &loop, &callback)) { + return -1; + } + + if (loop) { + Py_INCREF(loop); + self->loop = (libevwrapper_Loop *)loop; + } else { + return -1; + } + + if (callback) { + if (!PyCallable_Check(callback)) { + PyErr_SetString(PyExc_TypeError, "callback parameter must be callable"); + Py_XDECREF(loop); + return -1; + } + Py_INCREF(callback); + self->callback = callback; + } + ev_init(&self->timer, timer_callback); + self->timer.data = self; + return 0; +} + +static PyObject * +Timer_start(libevwrapper_Timer *self, PyObject *args) { + double timeout; + if (!PyArg_ParseTuple(args, "d", &timeout)) { + return NULL; + } + self->timer.repeat = fmax(timeout, 0.0); + ev_timer_again(self->loop->loop, &self->timer); + Py_RETURN_NONE; +} + +static PyObject * +Timer_stop(libevwrapper_Timer *self, PyObject *args) { + ev_timer_stop(self->loop->loop, &self->timer); + Py_RETURN_NONE; +} + +static PyMethodDef Timer_methods[] = { + {"start", (PyCFunction)Timer_start, METH_VARARGS, "Start the Timer watcher"}, + {"stop", (PyCFunction)Timer_stop, METH_NOARGS, "Stop the Timer watcher"}, + {NULL} /* Sentinal */ +}; + +static PyTypeObject libevwrapper_TimerType = { + PyVarObject_HEAD_INIT(NULL, 0) + "cassandra.io.libevwrapper.Timer", /*tp_name*/ + sizeof(libevwrapper_Timer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)Timer_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "Timer objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Timer_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)Timer_init, /* tp_init */ +}; + + static PyMethodDef module_methods[] = { {NULL} /* Sentinal */ }; @@ -500,6 +623,10 @@ initlibevwrapper(void) if (PyType_Ready(&libevwrapper_AsyncType) < 0) INITERROR; + libevwrapper_TimerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&libevwrapper_TimerType) < 0) + INITERROR; + # if PY_MAJOR_VERSION >= 3 module = PyModule_Create(&moduledef); # else @@ -532,6 +659,10 @@ initlibevwrapper(void) if (PyModule_AddObject(module, "Async", (PyObject *)&libevwrapper_AsyncType) == -1) INITERROR; + Py_INCREF(&libevwrapper_TimerType); + if (PyModule_AddObject(module, "Timer", (PyObject *)&libevwrapper_TimerType) == -1) + INITERROR; + if (!PyEval_ThreadsInitialized()) { PyEval_InitThreads(); } From 109d6bb8abe02789e5b78e46af5557e13bca73d5 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 10:44:14 -0500 Subject: [PATCH 0066/2431] Update ResponseFutures to use reactor timers PYTHON-108 avoids sleep in Event.wait, which contributes to latency when using synchronous execution --- cassandra/cluster.py | 66 ++++++++++++++++-------------- cassandra/query.py | 6 +-- tests/unit/test_response_future.py | 22 +++++----- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index c5c83a730f..834b96367b 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1401,17 +1401,14 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): trace details, the :attr:`~.Statement.trace` attribute will be left as :const:`None`. """ - if timeout is _NOT_SET: - timeout = self.default_timeout - if trace and not isinstance(query, Statement): raise TypeError( "The query argument must be an instance of a subclass of " "cassandra.query.Statement when trace=True") - future = self.execute_async(query, parameters, trace) + future = self.execute_async(query, parameters, trace, timeout) try: - result = future.result(timeout) + result = future.result() finally: if trace: try: @@ -1421,7 +1418,7 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): return result - def execute_async(self, query, parameters=None, trace=False): + def execute_async(self, query, parameters=None, trace=False, timeout=_NOT_SET): """ Execute the given query and return a :class:`~.ResponseFuture` object which callbacks may be attached to for asynchronous response @@ -1458,11 +1455,14 @@ def execute_async(self, query, parameters=None, trace=False): ... log.exception("Operation failed:") """ - future = self._create_response_future(query, parameters, trace) + if timeout is _NOT_SET: + timeout = self.default_timeout + + future = self._create_response_future(query, parameters, trace, timeout) future.send_request() return future - def _create_response_future(self, query, parameters, trace): + def _create_response_future(self, query, parameters, trace, timeout): """ Returns the ResponseFuture before calling send_request() on it """ prepared_statement = None @@ -1513,7 +1513,7 @@ def _create_response_future(self, query, parameters, trace): message.tracing = True return ResponseFuture( - self, message, query, self.default_timeout, metrics=self._metrics, + self, message, query, timeout, metrics=self._metrics, prepared_statement=prepared_statement) def prepare(self, query): @@ -1543,10 +1543,10 @@ def prepare(self, query): Preparing the same query more than once will likely affect performance. """ message = PrepareMessage(query=query) - future = ResponseFuture(self, message, query=None) + future = ResponseFuture(self, message, query=None, timeout=self.default_timeout) try: future.send_request() - query_id, column_metadata = future.result(self.default_timeout) + query_id, column_metadata = future.result() except Exception: log.exception("Error preparing query:") raise @@ -1571,7 +1571,7 @@ def prepare_on_all_hosts(self, query, excluded_host): futures = [] for host in self._pools.keys(): if host != excluded_host and host.is_up: - future = ResponseFuture(self, PrepareMessage(query=query), None) + future = ResponseFuture(self, PrepareMessage(query=query), None, self.default_timeout) # we don't care about errors preparing against specific hosts, # since we can always prepare them as needed when the prepared @@ -1592,7 +1592,7 @@ def prepare_on_all_hosts(self, query, excluded_host): for host, future in futures: try: - future.result(self.default_timeout) + future.result() except Exception: log.exception("Error preparing query for host %s:", host) @@ -2579,13 +2579,14 @@ class ResponseFuture(object): _start_time = None _metrics = None _paging_state = None + _timer = None - def __init__(self, session, message, query, default_timeout=None, metrics=None, prepared_statement=None): + def __init__(self, session, message, query, timeout, metrics=None, prepared_statement=None): self.session = session self.row_factory = session.row_factory self.message = message self.query = query - self.default_timeout = default_timeout + self.timeout = timeout self._metrics = metrics self.prepared_statement = prepared_statement self._callback_lock = Lock() @@ -2596,6 +2597,17 @@ def __init__(self, session, message, query, default_timeout=None, metrics=None, self._errors = {} self._callbacks = [] self._errbacks = [] + self._start_timer() + + def _start_timer(self): + self._timer = self.session.cluster.connection_class.create_timer(self.timeout, self._on_timeout) + + def _cancel_timer(self): + if self._timer: + self._timer.cancel() + + def _on_timeout(self): + self._set_final_exception(OperationTimedOut()) def _make_query_plan(self): # convert the list/generator/etc to an iterator so that subsequent @@ -2684,6 +2696,7 @@ def start_fetching_next_page(self): self._event.clear() self._final_result = _NOT_SET self._final_exception = None + self._start_timer() self.send_request() def _reprepare(self, prepare_message): @@ -2888,6 +2901,7 @@ def _execute_after_prepare(self, response): "statement on host %s: %s" % (self._current_host, response))) def _set_final_result(self, response): + self._cancel_timer() if self._metrics is not None: self._metrics.request_timer.addValue(time.time() - self._start_time) @@ -2902,6 +2916,7 @@ def _set_final_result(self, response): fn(response, *args, **kwargs) def _set_final_exception(self, response): + self._cancel_timer() if self._metrics is not None: self._metrics.request_timer.addValue(time.time() - self._start_time) @@ -2976,27 +2991,18 @@ def result(self, timeout=_NOT_SET): ... log.exception("Operation failed:") """ - if timeout is _NOT_SET: - timeout = self.default_timeout + if timeout is not _NOT_SET: + # TODO: warn deprecated + pass + self._event.wait() if self._final_result is not _NOT_SET: if self._paging_state is None: return self._final_result else: return PagedResult(self, self._final_result, timeout) - elif self._final_exception: - raise self._final_exception else: - self._event.wait(timeout=timeout) - if self._final_result is not _NOT_SET: - if self._paging_state is None: - return self._final_result - else: - return PagedResult(self, self._final_result, timeout) - elif self._final_exception: - raise self._final_exception - else: - raise OperationTimedOut(errors=self._errors, last_host=self._current_host) + raise self._final_exception def get_query_trace(self, max_wait=None): """ @@ -3162,7 +3168,7 @@ def next(self): raise self.response_future.start_fetching_next_page() - result = self.response_future.result(self.timeout) + result = self.response_future.result() if self.response_future.has_more_pages: self.current_response = result.current_response else: diff --git a/cassandra/query.py b/cassandra/query.py index 7f7756b3fb..3a5e867f9d 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -838,14 +838,14 @@ def populate(self, max_wait=2.0): break def _execute(self, query, parameters, time_spent, max_wait): + timeout = (max_wait - time_spent) if max_wait is not None else None + future = self._session._create_response_future(query, parameters, trace=False, timeout=timeout) # in case the user switched the row factory, set it to namedtuple for this query - future = self._session._create_response_future(query, parameters, trace=False) future.row_factory = named_tuple_factory future.send_request() - timeout = (max_wait - time_spent) if max_wait is not None else None try: - return future.result(timeout=timeout) + return future.result() except OperationTimedOut: raise TraceUnavailable("Trace information was not available within %f seconds" % (max_wait,)) diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index 027fe73214..986945ba4a 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -47,7 +47,7 @@ def make_session(self): def make_response_future(self, session): query = SimpleStatement("SELECT * FROM foo") message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - return ResponseFuture(session, message, query) + return ResponseFuture(session, message, query, 1) def make_mock_response(self, results): return Mock(spec=ResultMessage, kind=RESULT_KIND_ROWS, results=results, paging_state=None) @@ -122,7 +122,7 @@ def test_read_timeout_error_message(self): query.retry_policy.on_read_timeout.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=ReadTimeoutErrorMessage, info={}) @@ -137,7 +137,7 @@ def test_write_timeout_error_message(self): query.retry_policy.on_write_timeout.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=WriteTimeoutErrorMessage, info={}) @@ -151,7 +151,7 @@ def test_unavailable_error_message(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=UnavailableErrorMessage, info={}) @@ -165,7 +165,7 @@ def test_retry_policy_says_ignore(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.IGNORE, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=UnavailableErrorMessage, info={}) @@ -184,7 +184,7 @@ def test_retry_policy_says_retry(self): connection = Mock(spec=Connection) pool.borrow_connection.return_value = (connection, 1) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() rf.session._pools.get.assert_called_once_with('ip1') @@ -279,7 +279,7 @@ def test_all_pools_shutdown(self): session._load_balancer.make_query_plan.return_value = ['ip1', 'ip2'] session._pools.get.return_value.is_shutdown = True - rf = ResponseFuture(session, Mock(), Mock()) + rf = ResponseFuture(session, Mock(), Mock(), 1) rf.send_request() self.assertRaises(NoHostAvailable, rf.result) @@ -354,7 +354,7 @@ def test_errback(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() rf.add_errback(self.assertIsInstance, Exception) @@ -401,7 +401,7 @@ def test_multiple_errbacks(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() callback = Mock() @@ -431,7 +431,7 @@ def test_add_callbacks(self): message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) # test errback - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() rf.add_callbacks( @@ -443,7 +443,7 @@ def test_add_callbacks(self): self.assertRaises(Exception, rf.result) # test callback - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() callback = Mock() From 6ea1504e72f2ba83c4ab9d114dba214d593c265b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 11:16:18 -0500 Subject: [PATCH 0067/2431] Refactor redundant register_watcher[s] to Connection --- cassandra/connection.py | 24 +++++++++++++++++++----- cassandra/io/asyncorereactor.py | 12 ------------ cassandra/io/eventletreactor.py | 15 --------------- cassandra/io/geventreactor.py | 15 --------------- cassandra/io/libevreactor.py | 12 ------------ cassandra/io/twistedreactor.py | 21 --------------------- 6 files changed, 19 insertions(+), 80 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index f5ca774a0d..81ca7dbd52 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -39,7 +39,8 @@ QueryMessage, ResultMessage, decode_response, InvalidRequestException, SupportedMessage, AuthResponseMessage, AuthChallengeMessage, - AuthSuccessMessage, ProtocolException) + AuthSuccessMessage, ProtocolException, + RegisterMessage) from cassandra.util import OrderedDict @@ -372,11 +373,24 @@ def wait_for_responses(self, *msgs, **kwargs): self.defunct(exc) raise - def register_watcher(self, event_type, callback): - raise NotImplementedError() + def register_watcher(self, event_type, callback, register_timeout=None): + """ + Register a callback for a given event type. + """ + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), + timeout=register_timeout) - def register_watchers(self, type_callback_dict): - raise NotImplementedError() + def register_watchers(self, type_callback_dict, register_timeout=None): + """ + Register multiple callback/event type pairs, expressed as a dict. + """ + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), + timeout=register_timeout) def control_conn_disposed(self): self.is_control_connection = False diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index eeb319bb65..ee089ae4fd 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -40,7 +40,6 @@ from cassandra.connection import (Connection, ConnectionShutdown, ConnectionException, NONBLOCKING, Timer, TimerManager) -from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -318,14 +317,3 @@ def writable(self): def readable(self): return self._readable or (self.is_control_connection and not (self.is_defunct or self.is_closed)) - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), timeout=register_timeout) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 670d0f1865..131cb336cc 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -28,9 +28,7 @@ from six.moves import xrange -from cassandra import OperationTimedOut from cassandra.connection import Connection, ConnectionShutdown -from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -165,16 +163,3 @@ def push(self, data): chunk_size = self.out_buffer_size for i in xrange(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 6e9af0da4d..f315ad0506 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -25,9 +25,7 @@ from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -from cassandra import OperationTimedOut from cassandra.connection import Connection, ConnectionShutdown -from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -161,16 +159,3 @@ def push(self, data): chunk_size = self.out_buffer_size for i in xrange(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 174cfd859c..c4efbb0df5 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -24,7 +24,6 @@ from cassandra.connection import (Connection, ConnectionShutdown, NONBLOCKING, Timer, TimerManager) -from cassandra.protocol import RegisterMessage try: import cassandra.io.libevwrapper as libev except ImportError: @@ -395,14 +394,3 @@ def push(self, data): with self._deque_lock: self.deque.extend(chunks) self._libevloop.notify() - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), timeout=register_timeout) diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index ff81e5613f..d6534150c0 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -22,9 +22,7 @@ import weakref import atexit -from cassandra import OperationTimedOut from cassandra.connection import Connection, ConnectionShutdown -from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -220,22 +218,3 @@ def push(self, data): the event loop when it gets the chance. """ reactor.callFromThread(self.connector.transport.write, data) - - def register_watcher(self, event_type, callback, register_timeout=None): - """ - Register a callback for a given event type. - """ - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - """ - Register multiple callback/event type pairs, expressed as a dict. - """ - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) From 4d653576953fa2dffd901a009116730009974706 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 15:09:13 -0500 Subject: [PATCH 0068/2431] Gevent timer implementation --- cassandra/connection.py | 6 +++++- cassandra/io/geventreactor.py | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 81ca7dbd52..c21133fb74 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -925,6 +925,10 @@ def add_timer(self, timer): self._timers.put_nowait(timer) def service_timeouts(self): + """ + run callbacks on all expired timers + :return: next end time, or None + """ now = time.time() while not self._timers.empty(): timer = self._timers.get_nowait() @@ -932,7 +936,7 @@ def service_timeouts(self): timer.on_timeout() else: self._timers.put_nowait(timer) - break + return timer.end @property def next_timeout(self): diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index f315ad0506..076b13d99c 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -20,12 +20,13 @@ from functools import partial import logging import os +import time -from six.moves import xrange +from six.moves import range from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -from cassandra.connection import Connection, ConnectionShutdown +from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager log = logging.getLogger(__name__) @@ -48,6 +49,34 @@ class GeventConnection(Connection): _write_watcher = None _socket = None + _timers = None + _timeout_watcher = None + _new_timer = None + + @classmethod + def initialize_reactor(cls): + if not cls._timers: + cls._timers = TimerManager() + cls._timeout_watcher = gevent.spawn(cls.service_timeouts) + cls._new_timer = Event() + + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._timers.add_timer(timer) + cls._new_timer.set() + return timer + + @classmethod + def service_timeouts(cls): + timer_manager = cls._timers + timer_event = cls._new_timer + while True: + next_end = timer_manager.service_timeouts() + sleep_time = next_end - time.time() if next_end else 10000 + timer_event.wait(sleep_time) + timer_event.clear() + def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) @@ -157,5 +186,5 @@ def handle_read(self): def push(self, data): chunk_size = self.out_buffer_size - for i in xrange(0, len(data), chunk_size): + for i in range(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) From e3fdffb55cdb5176d22f5962594d30fee7616237 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 16:08:32 -0500 Subject: [PATCH 0069/2431] Catch and log exceptions during timeout servicing --- cassandra/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index c21133fb74..856cec31fd 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -933,7 +933,10 @@ def service_timeouts(self): while not self._timers.empty(): timer = self._timers.get_nowait() if timer.end < now: - timer.on_timeout() + try: + timer.on_timeout() + except Exception: + log.exception("Exception while servicing timeout callback: ") else: self._timers.put_nowait(timer) return timer.end From 87028134b1824ad767d511f6503c86a029cd7cb1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 30 Apr 2015 16:09:51 -0500 Subject: [PATCH 0070/2431] refactor connected_event to Connection init --- cassandra/connection.py | 1 + cassandra/io/asyncorereactor.py | 2 -- cassandra/io/geventreactor.py | 7 +++---- cassandra/io/libevreactor.py | 2 -- cassandra/io/twistedreactor.py | 1 - 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 856cec31fd..c1cb08e176 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -224,6 +224,7 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self._full_header_length = self._header_length + 4 self.lock = RLock() + self.connected_event = Event() @classmethod def initialize_reactor(self): diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index ee089ae4fd..8010b19237 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -158,8 +158,6 @@ def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) asyncore.dispatcher.__init__(self) - self.connected_event = Event() - self._callbacks = {} self.deque = deque() self.deque_lock = Lock() diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 076b13d99c..83abd5d81b 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -13,7 +13,7 @@ # limitations under the License. import gevent from gevent import select, socket, ssl -from gevent.event import Event +import gevent.event from gevent.queue import Queue from collections import defaultdict @@ -58,7 +58,7 @@ def initialize_reactor(cls): if not cls._timers: cls._timers = TimerManager() cls._timeout_watcher = gevent.spawn(cls.service_timeouts) - cls._new_timer = Event() + cls._new_timer = gevent.event.Event() @classmethod def create_timer(cls, timeout, callback): @@ -73,14 +73,13 @@ def service_timeouts(cls): timer_event = cls._new_timer while True: next_end = timer_manager.service_timeouts() - sleep_time = next_end - time.time() if next_end else 10000 + sleep_time = max(next_end - time.time(), 0) if next_end else 10000 timer_event.wait(sleep_time) timer_event.clear() def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() self._write_queue = Queue() self._callbacks = {} diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index c4efbb0df5..150fab21b5 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -264,8 +264,6 @@ def create_timer(self, timeout, callback): def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() - self._callbacks = {} self.deque = deque() self._deque_lock = Lock() diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index d6534150c0..1077cec0e8 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -157,7 +157,6 @@ def __init__(self, *args, **kwargs): """ Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() self.is_closed = True self.connector = None From 3489813bf7e66d39f51199c2c755e990a7d5c197 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 1 May 2015 09:32:24 -0500 Subject: [PATCH 0071/2431] Eventlet timer using eventlet.event.NOT_USED. This works with the current implementation, but there is no guarantee about the stability of this mechanism in eventlet events. Probably need to synchronize Event management. --- cassandra/io/eventletreactor.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 131cb336cc..709f3a44af 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -18,17 +18,18 @@ from collections import defaultdict from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -import eventlet +import eventlet.event from eventlet.green import select, socket from eventlet.queue import Queue +from eventlet.timeout import Timeout from functools import partial import logging import os -from threading import Event +import time from six.moves import xrange -from cassandra.connection import Connection, ConnectionShutdown +from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager log = logging.getLogger(__name__) @@ -51,14 +52,38 @@ class EventletConnection(Connection): _write_watcher = None _socket = None + _timers = None + _timeout_watcher = None + _new_timer = None + @classmethod def initialize_reactor(cls): eventlet.monkey_patch() + if not cls._timers: + cls._timers = TimerManager() + cls._timeout_watcher = eventlet.spawn(cls.service_timeouts) + cls._new_timer = eventlet.event.Event() + + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._timers.add_timer(timer) + cls._new_timer.send(eventlet.event.NOT_USED) + return timer + + @classmethod + def service_timeouts(cls): + timer_manager = cls._timers + while True: + next_end = timer_manager.service_timeouts() + sleep_time = max(next_end - time.time(), 0) if next_end else 10000 + with Timeout(sleep_time, False): + cls._new_timer.wait() + cls._new_timer = eventlet.event.Event() def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() self._write_queue = Queue() self._callbacks = {} From c44365c8b204edb778e2046d31e9f2ff193bb8a9 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 1 May 2015 10:42:06 -0500 Subject: [PATCH 0072/2431] Use threading.Event for eventlet timeouts Removes Eventlet Events, which are more difficult to use --- cassandra/io/eventletreactor.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 709f3a44af..8be3a07353 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -18,13 +18,13 @@ from collections import defaultdict from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -import eventlet.event +import eventlet from eventlet.green import select, socket from eventlet.queue import Queue -from eventlet.timeout import Timeout from functools import partial import logging import os +from threading import Event import time from six.moves import xrange @@ -62,13 +62,13 @@ def initialize_reactor(cls): if not cls._timers: cls._timers = TimerManager() cls._timeout_watcher = eventlet.spawn(cls.service_timeouts) - cls._new_timer = eventlet.event.Event() + cls._new_timer = Event() @classmethod def create_timer(cls, timeout, callback): timer = Timer(timeout, callback) cls._timers.add_timer(timer) - cls._new_timer.send(eventlet.event.NOT_USED) + cls._new_timer.set() return timer @classmethod @@ -77,9 +77,8 @@ def service_timeouts(cls): while True: next_end = timer_manager.service_timeouts() sleep_time = max(next_end - time.time(), 0) if next_end else 10000 - with Timeout(sleep_time, False): - cls._new_timer.wait() - cls._new_timer = eventlet.event.Event() + cls._new_timer.wait(sleep_time) + cls._new_timer.clear() def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) From 6ffca60cd1291277fd05967b9dbeecd764b9c615 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 1 May 2015 16:29:02 -0700 Subject: [PATCH 0073/2431] test for python-206 connect timeouts --- tests/integration/standard/test_cluster.py | 27 +++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index 0fcabd104e..146859ba81 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -30,7 +30,7 @@ WhiteListRoundRobinPolicy) from cassandra.query import SimpleStatement, TraceUnavailable -from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions +from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions, get_node from tests.integration.util import assert_quiescent_pool_state @@ -40,6 +40,31 @@ def setup_module(): class ClusterTests(unittest.TestCase): + def test_raise_error_on_control_connection_timeout(self): + """ + Test for initial control connection timeout + + test_raise_error_on_control_connection_timeout tests that the driver times out after the set initial connection + timeout. It first pauses node1, essentially making it unreachable. It then attempts to create a Cluster object + via connecting to node1 with a timeout of 1 second, and ensures that a NoHostAvailable is raised, along with + an OperationTimedOut for 1 second. + + @expected_errors NoHostAvailable When node1 is paused, and a connection attempt is made. + @since 2.6.0 + @jira_ticket PYTHON-206 + @expected_result NoHostAvailable exception should be raised after 1 second. + + @test_category connection + """ + + get_node(1).pause() + cluster = Cluster(contact_points=['127.0.0.1'], protocol_version=PROTOCOL_VERSION, connect_timeout=1) + + with self.assertRaisesRegexp(NoHostAvailable, "OperationTimedOut\('errors=Timed out creating connection \(1 seconds\)"): + cluster.connect() + + get_node(1).resume() + def test_basic(self): """ Test basic connection and usage From afe2e5ae8eabc101ae044869b21464d2733a4a14 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 4 May 2015 11:00:03 -0500 Subject: [PATCH 0074/2431] Twisted timer implementation --- cassandra/io/twistedreactor.py | 37 ++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index 1077cec0e8..12a8f5303f 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -15,14 +15,15 @@ Module that implements an event loop based on twisted ( https://twistedmatrix.com ). """ -from twisted.internet import reactor, protocol -from threading import Event, Thread, Lock +import atexit from functools import partial import logging +from threading import Event, Thread, Lock +import time +from twisted.internet import reactor, protocol import weakref -import atexit -from cassandra.connection import Connection, ConnectionShutdown +from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager log = logging.getLogger(__name__) @@ -107,9 +108,12 @@ class TwistedLoop(object): _lock = None _thread = None + _timeout_task = None + _timeout = None def __init__(self): self._lock = Lock() + self._timers = TimerManager() def maybe_start(self): with self._lock: @@ -131,6 +135,25 @@ def _cleanup(self): "Cluster.shutdown() to avoid this.") log.debug("Event loop thread was joined") + def add_timer(self, timer): + self._timers.add_timer(timer) + reactor.callFromThread(self._schedule_timeout, timer.end) + + def _schedule_timeout(self, next_timeout): + if next_timeout: + delay = max(next_timeout - time.time(), 0) + if self._timeout_task and self._timeout_task.active(): + if next_timeout < self._timeout: + self._timeout_task.reset(delay) + self._timeout = next_timeout + else: + self._timeout_task = reactor.callLater(delay, self._on_loop_timer) + self._timeout = next_timeout + + def _on_loop_timer(self): + self._timers.service_timeouts() + self._schedule_timeout(self._timers.next_timeout) + class TwistedConnection(Connection): """ @@ -146,6 +169,12 @@ def initialize_reactor(cls): if not cls._loop: cls._loop = TwistedLoop() + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._loop.add_timer(timer) + return timer + def __init__(self, *args, **kwargs): """ Initialization method. From 7f5e9df1b6f27629e280e0d9947950565d55af90 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 4 May 2015 11:00:39 -0500 Subject: [PATCH 0075/2431] self-->cls in create_timer classmethods --- cassandra/io/asyncorereactor.py | 4 ++-- cassandra/io/libevreactor.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 8010b19237..52972b9e45 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -149,9 +149,9 @@ def handle_fork(cls): cls._loop = None @classmethod - def create_timer(self, timeout, callback): + def create_timer(cls, timeout, callback): timer = Timer(timeout, callback) - self._loop.add_timer(timer) + cls._loop.add_timer(timer) return timer def __init__(self, *args, **kwargs): diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 150fab21b5..3559c0bf59 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -256,9 +256,9 @@ def handle_fork(cls): cls._libevloop = None @classmethod - def create_timer(self, timeout, callback): + def create_timer(cls, timeout, callback): timer = Timer(timeout, callback) - self._libevloop.add_timer(timer) + cls._libevloop.add_timer(timer) return timer def __init__(self, *args, **kwargs): From 67e19aad0b7ce13764bb1c7460366392c20fef1f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 4 May 2015 12:01:16 -0500 Subject: [PATCH 0076/2431] Refactor _callbacks and _push_watchers init to Connection --- cassandra/connection.py | 1 + cassandra/io/asyncorereactor.py | 1 - cassandra/io/eventletreactor.py | 3 --- cassandra/io/geventreactor.py | 3 --- cassandra/io/libevreactor.py | 1 - cassandra/io/twistedreactor.py | 1 - tests/unit/test_connection.py | 3 --- 7 files changed, 1 insertion(+), 12 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index c1cb08e176..57f5f96ad8 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -194,6 +194,7 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.is_control_connection = is_control_connection self.user_type_map = user_type_map self._push_watchers = defaultdict(set) + self._callbacks = {} self._iobuf = io.BytesIO() if protocol_version >= 3: self._header_unpack = v3_header_unpack diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 52972b9e45..b815d20aed 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -158,7 +158,6 @@ def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) asyncore.dispatcher.__init__(self) - self._callbacks = {} self.deque = deque() self.deque_lock = Lock() diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 8be3a07353..5db0816c88 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -85,9 +85,6 @@ def __init__(self, *args, **kwargs): self._write_queue = Queue() - self._callbacks = {} - self._push_watchers = defaultdict(set) - sockerr = None addresses = socket.getaddrinfo( self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 83abd5d81b..84285957bb 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -82,9 +82,6 @@ def __init__(self, *args, **kwargs): self._write_queue = Queue() - self._callbacks = {} - self._push_watchers = defaultdict(set) - sockerr = None addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) for (af, socktype, proto, canonname, sockaddr) in addresses: diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 3559c0bf59..31c9cd3812 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -264,7 +264,6 @@ def create_timer(cls, timeout, callback): def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self._callbacks = {} self.deque = deque() self._deque_lock = Lock() diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index 12a8f5303f..9b835fe137 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -189,7 +189,6 @@ def __init__(self, *args, **kwargs): self.is_closed = True self.connector = None - self._callbacks = {} reactor.callFromThread(self.add_connection) self._loop.maybe_start() diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 7fc4ed4e5a..a779e1338c 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -261,10 +261,7 @@ def test_not_implemented(self): Ensure the following methods throw NIE's. If not, come back and test them. """ c = self.make_connection() - self.assertRaises(NotImplementedError, c.close) - self.assertRaises(NotImplementedError, c.register_watcher, None, None) - self.assertRaises(NotImplementedError, c.register_watchers, None) def test_set_keyspace_blocking(self): c = self.make_connection() From d7c68f9e18178c9d1f07e3c49348b27f94d1afe8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 4 May 2015 13:23:29 -0500 Subject: [PATCH 0077/2431] Allow ResponseFutures without a timeout --- cassandra/cluster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 834b96367b..09ce11f657 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2600,7 +2600,8 @@ def __init__(self, session, message, query, timeout, metrics=None, prepared_stat self._start_timer() def _start_timer(self): - self._timer = self.session.cluster.connection_class.create_timer(self.timeout, self._on_timeout) + if self.timeout is not None: + self._timer = self.session.cluster.connection_class.create_timer(self.timeout, self._on_timeout) def _cancel_timer(self): if self._timer: From 4b2f046452826119b03ed17b110bc49e6af30e0f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 4 May 2015 13:24:21 -0500 Subject: [PATCH 0078/2431] Remove deprecated, unused import from cqlengine getting started --- docs/object_mapper.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/object_mapper.rst b/docs/object_mapper.rst index 26d78a0964..4e38994064 100644 --- a/docs/object_mapper.rst +++ b/docs/object_mapper.rst @@ -48,7 +48,7 @@ Getting Started from cassandra.cqlengine import columns from cassandra.cqlengine import connection from datetime import datetime - from cassandra.cqlengine.management import create_keyspace, sync_table + from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.models import Model #first, define a model From 0356f85c06570ca755ac9ae3f2e83c7db8b84888 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Mon, 4 May 2015 15:02:59 -0700 Subject: [PATCH 0079/2431] tests for PYTHON-246 --- .../cqlengine/model/test_model_io.py | 101 +++++++++++++++++- .../integration/cqlengine/model/test_udts.py | 72 +++++++++---- 2 files changed, 151 insertions(+), 22 deletions(-) diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index b9f3936f19..1af9a8af0f 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from uuid import uuid4 +from uuid import uuid4, UUID import random -from datetime import date +from datetime import date, datetime +from decimal import Decimal from operator import itemgetter from cassandra.cqlengine import CQLEngineException from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -124,6 +125,102 @@ def test_a_sensical_error_is_raised_if_you_try_to_create_a_table_twice(self): sync_table(TestModel) sync_table(TestModel) + def test_can_insert_model_with_all_column_types(self): + """ + Test for inserting all column types into a Model + + test_can_insert_model_with_all_column_types tests that each cqlengine column type can be inserted into a Model. + It first creates a Model that has each cqlengine column type. It then creates a Model instance where all the fields + have corresponding data, which performs the insert into the Cassandra table. + Finally, it verifies that each column read from the Model from Cassandra is the same as the input parameters. + + @since 2.6.0 + @jira_ticket PYTHON-246 + @expected_result The Model is inserted with each column type, and the resulting read yields proper data for each column. + + @test_category data_types:primitive + """ + + class AllDatatypesModel(Model): + id = columns.Integer(primary_key=True) + a = columns.Ascii() + b = columns.BigInt() + c = columns.Blob() + d = columns.Boolean() + e = columns.Date() + f = columns.DateTime() + g = columns.Decimal() + h = columns.Double() + i = columns.Float(double_precision=False) + j = columns.Inet() + k = columns.Integer() + l = columns.Text() + m = columns.TimeUUID() + n = columns.UUID() + o = columns.VarInt() + + sync_table(AllDatatypesModel) + + input = ['ascii', 2 ** 63 - 1, bytearray(b'hello world'), True, date(1970, 1, 1), + datetime.utcfromtimestamp(872835240), Decimal('12.3E+7'), 2.39, + 3.4028234663852886e+38, '123.123.123.123', 2147483647, 'text', + UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + int(str(2147483647) + '000')] + + AllDatatypesModel.create(id=0, a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1), + f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, + i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l='text', + m=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), n=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + o=int(str(2147483647) + '000')) + + self.assertEqual(1, AllDatatypesModel.objects.count()) + output = AllDatatypesModel.objects().first() + + for i, i_char in enumerate(range(ord('a'), ord('a') + 15)): + self.assertEqual(input[i], output[chr(i_char)]) + + def test_can_insert_double_and_float(self): + """ + Test for inserting single-precision and double-precision values into a Float and Double columns + + test_can_insert_double_and_float tests a Float can only hold a single-precision value, unless + "double_precision" attribute is specified as True or is unspecified. This test first tests that an AttributeError + is raised when attempting to input a double-precision value into a single-precision Float. It then verifies that + Double, Float(double_precision=True) and Float() can hold double-precision values by default. It also verifies that + columns.Float(double_precision=False) can hold a single-precision value, and a Double can hold a single-precision value. + + @since 2.6.0 + @jira_ticket PYTHON-246 + @expected_result Each floating point column type is able to hold their respective precision values. + + @test_category data_types:primitive + """ + + class FloatingPointModel(Model): + id = columns.Integer(primary_key=True) + a = columns.Float(double_precision=False) + b = columns.Float(double_precision=True) + c = columns.Float() + d = columns.Double() + + sync_table(FloatingPointModel) + + FloatingPointModel.create(id=0, a=2.39) + output = FloatingPointModel.objects().first() + self.assertEqual(2.390000104904175, output.a) + + FloatingPointModel.create(id=0, a=3.4028234663852886e+38, b=2.39, c=2.39, d=2.39) + output = FloatingPointModel.objects().first() + + self.assertEqual(3.4028234663852886e+38, output.a) + self.assertEqual(2.39, output.b) + self.assertEqual(2.39, output.c) + self.assertEqual(2.39, output.d) + + FloatingPointModel.create(id=0, d=3.4028234663852886e+38) + output = FloatingPointModel.objects().first() + self.assertEqual(3.4028234663852886e+38, output.d) + class TestMultiKeyModel(Model): diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 89a55d1730..3299fda310 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -187,7 +187,22 @@ class DepthModel(Model): self.assertEqual(udts[2], output.v_2) self.assertEqual(udts[3], output.v_3) - def test_can_insert_udts_with_nulls(self): + def test_can_insert_udts_with_nones(self): + """ + Test for inserting all column types as empty into a UserType as None's + + test_can_insert_udts_with_nones tests that each cqlengine column type can be inserted into a UserType as None's. + It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates + a UserType instance where all the fields are None's and inserts the UserType as an instance of the Model. Finally, + it verifies that each column read from the UserType from Cassandra is None. + + @since 2.5.0 + @jira_ticket PYTHON-251 + @expected_result The UserType is inserted with each column type, and the resulting read yields None's for each column. + + @test_category data_types:udt + """ + class AllDatatypes(UserType): a = columns.Ascii() b = columns.BigInt() @@ -196,13 +211,14 @@ class AllDatatypes(UserType): e = columns.Date() f = columns.DateTime() g = columns.Decimal() - h = columns.Float(double_precision=False) - i = columns.Inet() - j = columns.Integer() - k = columns.Text() - l = columns.TimeUUID() - m = columns.UUID() - n = columns.VarInt() + h = columns.Double() + i = columns.Float(double_precision=False) + j = columns.Inet() + k = columns.Integer() + l = columns.Text() + m = columns.TimeUUID() + n = columns.UUID() + o = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) @@ -219,6 +235,21 @@ class AllDatatypesModel(Model): self.assertEqual(input, output) def test_can_insert_udts_with_all_datatypes(self): + """ + Test for inserting all column types into a UserType + + test_can_insert_udts_with_all_datatypes tests that each cqlengine column type can be inserted into a UserType. + It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates + a UserType instance where all the fields have corresponding data, and inserts the UserType as an instance of the Model. + Finally, it verifies that each column read from the UserType from Cassandra is the same as the input parameters. + + @since 2.5.0 + @jira_ticket PYTHON-251 + @expected_result The UserType is inserted with each column type, and the resulting read yields proper data for each column. + + @test_category data_types:udt + """ + class AllDatatypes(UserType): a = columns.Ascii() b = columns.BigInt() @@ -228,12 +259,13 @@ class AllDatatypes(UserType): f = columns.DateTime() g = columns.Decimal() h = columns.Double() - i = columns.Inet() - j = columns.Integer() - k = columns.Text() - l = columns.TimeUUID() - m = columns.UUID() - n = columns.VarInt() + i = columns.Float(double_precision=False) + j = columns.Inet() + k = columns.Integer() + l = columns.Text() + m = columns.TimeUUID() + n = columns.UUID() + o = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) @@ -242,14 +274,14 @@ class AllDatatypesModel(Model): sync_table(AllDatatypesModel) input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1), - f=datetime.utcfromtimestamp(872835240), - g=Decimal('12.3E+7'), h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647, - k='text', l= UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), - m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000')) - alldata = AllDatatypesModel.create(id=0, data=input) + f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, + i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l='text', + m=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), n=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + o=int(str(2147483647) + '000')) + AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first().data - for i in range(ord('a'), ord('a') + 14): + for i in range(ord('a'), ord('a') + 15): self.assertEqual(input[chr(i)], output[chr(i)]) From 7cb178ff0190b785369747d42604eee64ad6bbaf Mon Sep 17 00:00:00 2001 From: GregBestland Date: Tue, 5 May 2015 17:24:40 -0500 Subject: [PATCH 0080/2431] Adding tests for PYTHON-235 --- tests/integration/standard/test_query.py | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 780f307891..45cdcaf880 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os - +import socket from cassandra.concurrent import execute_concurrent @@ -89,6 +89,44 @@ def test_trace_ignores_row_factory(self): cluster.shutdown() + def test_trace_client_ip(self): + """ ++ Test to validate that client trace contains client ip information. ++ ++ creates a simple query and ensures that the client trace information is present. This will + only be the case if the CQL version is 4 or greater./ + ++ ++ @since 3.0 ++ @jira_ticket PYTHON-235 ++ @expected_result client address should be present in CQL > 4, otherwise should be none. ++ ++ @test_category trace ++ """ + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest( + "Protocol 4+ is required for client ip tracing, currently testing against %r" + % (PROTOCOL_VERSION,)) + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + session = cluster.connect() + + query = "SELECT * FROM system.local" + statement = SimpleStatement(query) + session.execute(statement, trace=True) + + # Fetch the client_ip from the trace. + client_ip=statement.trace.client + # Ensure that ip is set for CQL >4 + self.assertIsNotNone(client_ip,"Client IP was not set in trace with CQL >=4.0") + # TODO we might want validate that client_ip actually matches our local ip rather than just validate that it + # is a valid ip. + try: + socket.inet_aton(client_ip) + except socket.error: + self.fail("Client IP retrieved from trace was not valid :{0}".format(client_ip)) + + cluster.shutdown() + class PreparedStatementTests(unittest.TestCase): From 60f59ce7f29f4d23fba974b4fb022adb3acf8465 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 6 May 2015 10:50:05 -0500 Subject: [PATCH 0081/2431] Allow canceled timers to finish ahead of end time. --- cassandra/connection.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 57f5f96ad8..54641a3ac5 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -911,8 +911,11 @@ def cancel(self): self.callback = self._noop self.cancelled = True - def on_timeout(self): - self.callback() + def finish(self, time_now): + if self.callback is self._noop or time_now >= self.end: + self.callback() + return True + return False def _noop(self): pass @@ -934,14 +937,12 @@ def service_timeouts(self): now = time.time() while not self._timers.empty(): timer = self._timers.get_nowait() - if timer.end < now: - try: - timer.on_timeout() - except Exception: - log.exception("Exception while servicing timeout callback: ") - else: - self._timers.put_nowait(timer) - return timer.end + try: + if not timer.finish(now): + self._timers.put_nowait(timer) + return timer.end + except Exception: + log.exception("Exception while servicing timeout callback: ") @property def next_timeout(self): From c17e6e45e3d5b18961f59ccccd38ce71eee2cf0a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 6 May 2015 10:52:04 -0500 Subject: [PATCH 0082/2431] libev: service timeouts before updating loop timer. Also, never start a timer with zero repeat (this causes it to stop, rather than timeout immediately). --- cassandra/io/libevreactor.py | 1 + cassandra/io/libevwrapper.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 31c9cd3812..e6abb76b41 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -148,6 +148,7 @@ def add_timer(self, timer): def _update_timer(self): if not self._shutdown: + self._timers.service_timeouts() offset = self._timers.next_offset or 100000 # none pending; will be updated again when something new happens self._loop_timer.start(offset) else: diff --git a/cassandra/io/libevwrapper.c b/cassandra/io/libevwrapper.c index 1c52a96464..99e1df30f7 100644 --- a/cassandra/io/libevwrapper.c +++ b/cassandra/io/libevwrapper.c @@ -517,7 +517,9 @@ Timer_start(libevwrapper_Timer *self, PyObject *args) { if (!PyArg_ParseTuple(args, "d", &timeout)) { return NULL; } - self->timer.repeat = fmax(timeout, 0.0); + /* some tiny non-zero number to avoid zero, and + make it run immediately for negative timeouts */ + self->timer.repeat = fmax(timeout, 0.000000001); ev_timer_again(self->loop->loop, &self->timer); Py_RETURN_NONE; } From bca2a69392175a1b15f2a8c32bf1fc6c67f3d758 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Wed, 6 May 2015 13:07:06 -0500 Subject: [PATCH 0083/2431] Fixing test for PYTHON-235 --- tests/integration/standard/test_query.py | 57 +++++++++++++----------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 45cdcaf880..bb1cafa776 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -27,7 +27,7 @@ from cassandra.cluster import Cluster from cassandra.policies import HostDistance -from tests.integration import use_singledc, PROTOCOL_VERSION +from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions def setup_module(): @@ -89,42 +89,45 @@ def test_trace_ignores_row_factory(self): cluster.shutdown() - def test_trace_client_ip(self): + def test_client_ip_in_trace(self): """ -+ Test to validate that client trace contains client ip information. -+ -+ creates a simple query and ensures that the client trace information is present. This will - only be the case if the CQL version is 4 or greater./ + Test to validate that client trace contains client ip information. -+ -+ @since 3.0 -+ @jira_ticket PYTHON-235 -+ @expected_result client address should be present in CQL > 4, otherwise should be none. -+ -+ @test_category trace + creates a simple query and ensures that the client trace information is present. This will + only be the case if the c* version is 3.0 or greater + + + @since 3.0 + @jira_ticket PYTHON-235 + @expected_result client address should be present in C* > 3, otherwise should be none. + + @test_category tracing + """ + #The current version on the trunk doesn't have the version set to 3.0 yet. + #For now we will use the protocol version. Once they update the version on C* trunk + #we can use the C*. See below + #self._cass_version, self._cql_version = get_server_versions() + #if self._cass_version < (3, 0): + # raise unittest.SkipTest("Client IP was not present in trace until C* 3.0") if PROTOCOL_VERSION < 4: - raise unittest.SkipTest( - "Protocol 4+ is required for client ip tracing, currently testing against %r" - % (PROTOCOL_VERSION,)) + raise unittest.SkipTest( + "Protocol 4+ is required for client ip tracing, currently testing against %r" + % (PROTOCOL_VERSION,)) + cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() query = "SELECT * FROM system.local" statement = SimpleStatement(query) - session.execute(statement, trace=True) - + response_future = session.execute_async(statement, trace=True) + response_future.result(10.0) + current_host = response_future._current_host.address # Fetch the client_ip from the trace. - client_ip=statement.trace.client - # Ensure that ip is set for CQL >4 - self.assertIsNotNone(client_ip,"Client IP was not set in trace with CQL >=4.0") - # TODO we might want validate that client_ip actually matches our local ip rather than just validate that it - # is a valid ip. - try: - socket.inet_aton(client_ip) - except socket.error: - self.fail("Client IP retrieved from trace was not valid :{0}".format(client_ip)) - + trace = response_future.get_query_trace(2.0) + client_ip = trace.client + # Ensure that ip is set for c* >3 + self.assertIsNotNone(client_ip,"Client IP was not set in trace with C* > 3.0") + self.assertEqual(client_ip,current_host,"Client IP from trace did not match the expected value") cluster.shutdown() From 9d8c50bb980f3611cb5e4705adbb3e036e48ef11 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 6 May 2015 14:53:07 -0500 Subject: [PATCH 0084/2431] Deprecate ResponseFuture.result timeout --- cassandra/cluster.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 09ce11f657..573ba3db64 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -27,6 +27,7 @@ import sys import time from threading import Lock, RLock, Thread, Event +import warnings import six from six.moves import range @@ -71,6 +72,7 @@ BatchStatement, bind_params, QueryTrace, Statement, named_tuple_factory, dict_factory, FETCH_SIZE_UNSET) + def _is_eventlet_monkey_patched(): if 'eventlet.patcher' not in sys.modules: return False @@ -1265,8 +1267,7 @@ class Session(object): """ A default timeout, measured in seconds, for queries executed through :meth:`.execute()` or :meth:`.execute_async()`. This default may be - overridden with the `timeout` parameter for either of those methods - or the `timeout` parameter for :meth:`.ResponseFuture.result()`. + overridden with the `timeout` parameter for either of those methods. Setting this to :const:`None` will cause no timeouts to be set by default. @@ -2961,6 +2962,11 @@ def result(self, timeout=_NOT_SET): encountered. If the final result or error has not been set yet, this method will block until that time. + .. versionchanged:: 2.6.0 + + **`timeout` is deprecated. Use timeout in the Session execute functions instead. + The following description applies to deprecated behavior:** + You may set a timeout (in seconds) with the `timeout` parameter. By default, the :attr:`~.default_timeout` for the :class:`.Session` this was created through will be used for the timeout on this @@ -2993,10 +2999,13 @@ def result(self, timeout=_NOT_SET): """ if timeout is not _NOT_SET: - # TODO: warn deprecated - pass + msg = "ResponseFuture.result timeout argument is deprecated. Specify the request timeout via Session.execute[_async]." + warnings.warn(msg, DeprecationWarning) + log.warning(msg) + else: + timeout = None - self._event.wait() + self._event.wait(timeout) if self._final_result is not _NOT_SET: if self._paging_state is None: return self._final_result From 70478e62fd7d98823c1ad2dd9e6dcb96dd4c0601 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 6 May 2015 15:01:14 -0500 Subject: [PATCH 0085/2431] Remove unused 'timeout' in ..cluster.PagedResult --- cassandra/cluster.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 573ba3db64..b8a236a30e 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -3010,7 +3010,7 @@ def result(self, timeout=_NOT_SET): if self._paging_state is None: return self._final_result else: - return PagedResult(self, self._final_result, timeout) + return PagedResult(self, self._final_result) else: raise self._final_exception @@ -3162,10 +3162,9 @@ class will be returned. response_future = None - def __init__(self, response_future, initial_response, timeout=_NOT_SET): + def __init__(self, response_future, initial_response): self.response_future = response_future self.current_response = iter(initial_response) - self.timeout = timeout def __iter__(self): return self From 7f2e992d768fbcfee57e493ded850a9299f9408c Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 6 May 2015 16:16:40 -0500 Subject: [PATCH 0086/2431] Minor tweaks on trace client integration test --- tests/integration/standard/test_query.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index bb1cafa776..7f84f9ab31 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import socket from cassandra.concurrent import execute_concurrent @@ -99,7 +98,7 @@ def test_client_ip_in_trace(self): @since 3.0 @jira_ticket PYTHON-235 - @expected_result client address should be present in C* > 3, otherwise should be none. + @expected_result client address should be present in C* >= 3, otherwise should be none. @test_category tracing + """ From 087293050b322b736a7468ae70c2044d1510c47a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 6 May 2015 16:29:23 -0500 Subject: [PATCH 0087/2431] Update unit tests for new refresh_schema signature. --- tests/unit/test_control_connection.py | 6 +++--- tests/unit/test_response_future.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 1f1ccc1bf1..9c93f5afe4 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -410,12 +410,12 @@ def test_handle_schema_change(self): } self.cluster.scheduler.reset_mock() self.control_connection._handle_schema_change(event) - self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', 'table1', None) + self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', 'table1', None, None, None) self.cluster.scheduler.reset_mock() event['table'] = None self.control_connection._handle_schema_change(event) - self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', None, None) + self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', None, None, None, None) def test_refresh_disabled(self): cluster = MockCluster() @@ -463,4 +463,4 @@ def test_refresh_disabled(self): cc_no_topo_refresh._handle_schema_change(schema_event) cluster.scheduler.schedule_unique.assert_has_calls([call(ANY, cc_no_topo_refresh.refresh_node_list_and_token_map), call(0.0, cc_no_topo_refresh.refresh_schema, - schema_event['keyspace'], schema_event['table'], None)]) + schema_event['keyspace'], schema_event['table'], None, None, None)]) diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index 027fe73214..92351a9d1d 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -105,7 +105,7 @@ def test_schema_change_result(self): kind=RESULT_KIND_SCHEMA_CHANGE, results={'keyspace': "keyspace1", "table": "table1"}) rf._set_result(result) - session.submit.assert_called_once_with(ANY, 'keyspace1', 'table1', None, ANY, rf) + session.submit.assert_called_once_with(ANY, 'keyspace1', 'table1', None, None, None, ANY, rf) def test_other_result_message_kind(self): session = self.make_session() From b10d945886a84e73b9542f389d9a46a8e717775c Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 7 May 2015 15:59:35 -0500 Subject: [PATCH 0088/2431] Synchronize, and short circuit CC._signal_error during shutdown Addresses an issue where a failed executor-scheduled refresh_schema logs an error because self._connection is removed during shutdown. --- cassandra/cluster.py | 36 +++++++++++++++++++---------------- tests/integration/__init__.py | 2 +- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index c5c83a730f..f1c6cd3615 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2009,14 +2009,14 @@ def shutdown(self): else: self._is_shutdown = True - log.debug("Shutting down control connection") - # stop trying to reconnect (if we are) - if self._reconnection_handler: - self._reconnection_handler.cancel() + log.debug("Shutting down control connection") + # stop trying to reconnect (if we are) + if self._reconnection_handler: + self._reconnection_handler.cancel() - if self._connection: - self._connection.close() - del self._connection + if self._connection: + self._connection.close() + del self._connection def refresh_schema(self, keyspace=None, table=None, usertype=None, schema_agreement_wait=None): @@ -2390,17 +2390,21 @@ def _get_schema_mismatches(self, peers_result, local_result, local_address): return dict((version, list(nodes)) for version, nodes in six.iteritems(versions)) def _signal_error(self): - # try just signaling the cluster, as this will trigger a reconnect - # as part of marking the host down - if self._connection and self._connection.is_defunct: - host = self._cluster.metadata.get_host(self._connection.host) - # host may be None if it's already been removed, but that indicates - # that errors have already been reported, so we're fine - if host: - self._cluster.signal_connection_failure( - host, self._connection.last_error, is_host_addition=False) + with self._lock: + if self._is_shutdown: return + # try just signaling the cluster, as this will trigger a reconnect + # as part of marking the host down + if self._connection and self._connection.is_defunct: + host = self._cluster.metadata.get_host(self._connection.host) + # host may be None if it's already been removed, but that indicates + # that errors have already been reported, so we're fine + if host: + self._cluster.signal_connection_failure( + host, self._connection.last_error, is_host_addition=False) + return + # if the connection is not defunct or the host already left, reconnect # manually self.reconnect() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 4b3c538145..54e785facc 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -228,7 +228,7 @@ def teardown_package(): log.exception('Failed to remove cluster: %s' % cluster_name) except Exception: - log.warn('Did not find cluster: %s' % cluster_name) + log.warning('Did not find cluster: %s' % cluster_name) def setup_test_keyspace(ipformat=None): From 36492777b32aecf358892144b4aca1159a907428 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 7 May 2015 16:32:04 -0500 Subject: [PATCH 0089/2431] load_balancing_policy defaults to TokeanAware(DCAware) PYTHON-160 --- cassandra/cluster.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index f1c6cd3615..436bfd929e 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -61,7 +61,7 @@ RESULT_KIND_SET_KEYSPACE, RESULT_KIND_ROWS, RESULT_KIND_SCHEMA_CHANGE) from cassandra.metadata import Metadata, protect_name -from cassandra.policies import (RoundRobinPolicy, SimpleConvictionPolicy, +from cassandra.policies import (TokenAwarePolicy, DCAwareRoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, RetryPolicy) from cassandra.pool import (Host, _ReconnectionHandler, _HostReconnectionHandler, @@ -161,6 +161,16 @@ def _shutdown_cluster(cluster): cluster.shutdown() +# murmur3 implementation required for TokenAware is only available for CPython +import platform +if platform.python_implementation() == 'CPython': + def default_lbp_factory(): + return TokenAwarePolicy(DCAwareRoundRobinPolicy()) +else: + def default_lbp_factory(): + return DCAwareRoundRobinPolicy() + + class Cluster(object): """ The main class to use when interacting with a Cassandra cluster. @@ -185,9 +195,9 @@ class Cluster(object): Defaults to loopback interface. Note: When using :class:`.DCAwareLoadBalancingPolicy` with no explicit - local_dc set, the DC is chosen from an arbitrary host in contact_points. - In this case, contact_points should contain only nodes from a single, - local DC. + local_dc set (as is the default), the DC is chosen from an arbitrary + host in contact_points. In this case, contact_points should contain + only nodes from a single, local DC. """ port = 9042 @@ -281,7 +291,16 @@ def auth_provider(self, value): load_balancing_policy = None """ An instance of :class:`.policies.LoadBalancingPolicy` or - one of its subclasses. Defaults to :class:`~.RoundRobinPolicy`. + one of its subclasses. + + .. versionchanged:: 2.6.0 + + Defaults to :class:`~.TokenAwarePolicy` (:class:`~.DCAwareRoundRobinPolicy`). + when using CPython (where the murmur3 extension is available). :class:`~.DCAwareRoundRobinPolicy` + otherwise. Default local DC will be chosen from contact points. + + **Please see** :class:`~.DCAwareRoundRobinPolicy` **for a discussion on default behavior with respect to + DC locality and remote nodes.** """ reconnection_policy = ExponentialReconnectionPolicy(1.0, 600.0) @@ -311,6 +330,8 @@ def auth_provider(self, value): by the :attr:`~.Cluster.load_balancing_policy` will have a connection opened to them. Otherwise, they will not have a connection opened to them. + Note that the default load balancing policy ignores remote hosts by default. + .. versionadded:: 2.1.0 """ @@ -495,7 +516,7 @@ def __init__(self, self.load_balancing_policy = load_balancing_policy else: - self.load_balancing_policy = RoundRobinPolicy() + self.load_balancing_policy = default_lbp_factory() if reconnection_policy is not None: if isinstance(reconnection_policy, type): From ce15713f0898fdff2f4b5006c55b9e8c00b3458a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 7 May 2015 16:51:07 -0500 Subject: [PATCH 0090/2431] Remove noop from Timer class, use flag. --- cassandra/connection.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 54641a3ac5..435f1e6f75 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -896,29 +896,29 @@ def _raise_if_stopped(self): class Timer(object): - cancelled = False + canceled = False def __init__(self, timeout, callback): self.end = time.time() + timeout self.callback = callback if timeout < 0: - self.on_timeout() + self.callback() def __lt__(self, other): return self.end < other.end def cancel(self): - self.callback = self._noop - self.cancelled = True + self.canceled = True def finish(self, time_now): - if self.callback is self._noop or time_now >= self.end: + if self.canceled: + return True + + if time_now >= self.end: self.callback() return True - return False - def _noop(self): - pass + return False class TimerManager(object): From fab252154efa23adceecef1f0403c69f0d937813 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 8 May 2015 10:04:58 -0500 Subject: [PATCH 0091/2431] Explanation of eventletreactor service_timeouts routine --- cassandra/io/eventletreactor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 5db0816c88..aceac55e10 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -16,7 +16,6 @@ # Originally derived from MagnetoDB source: # https://github.com/stackforge/magnetodb/blob/2015.1.0b1/magnetodb/common/cassandra/io/eventletreactor.py -from collections import defaultdict from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL import eventlet from eventlet.green import select, socket @@ -73,6 +72,12 @@ def create_timer(cls, timeout, callback): @classmethod def service_timeouts(cls): + """ + cls._timeout_watcher runs in this loop forever. + It is usually waiting for the next timeout on the cls._new_timer Event. + When new timers are added, that event is set so that the watcher can + wake up and possibly set an earlier timeout. + """ timer_manager = cls._timers while True: next_end = timer_manager.service_timeouts() From 2d3e33068071142c59bdaed8c1c6b18152d43d62 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 8 May 2015 11:01:51 -0500 Subject: [PATCH 0092/2431] Explain TwistedLoop add_timer thread interaction --- cassandra/io/twistedreactor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index 9b835fe137..0f5c841c75 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -137,6 +137,8 @@ def _cleanup(self): def add_timer(self, timer): self._timers.add_timer(timer) + # callFromThread to schedule from the loop thread, where + # the timeout task can safely be modified reactor.callFromThread(self._schedule_timeout, timer.end) def _schedule_timeout(self, next_timeout): From 7e70d5543d08035ff7a4a94a7e9564f5ffc7281c Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 09:53:18 -0500 Subject: [PATCH 0093/2431] Coarse locking on TimerManager.service_timeouts --- cassandra/connection.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 435f1e6f75..707a2a4db0 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -16,11 +16,12 @@ from collections import defaultdict, deque import errno from functools import wraps, partial +from heapq import heappush, heappop import io import logging import os import sys -from threading import Thread, Event, RLock +from threading import Thread, Event, RLock, Lock import time if 'gevent.monkey' in sys.modules: @@ -29,7 +30,6 @@ from six.moves.queue import Queue, Empty # noqa import six -from six.moves import queue from six.moves import range from cassandra import ConsistencyLevel, AuthenticationFailed, OperationTimedOut @@ -924,10 +924,12 @@ def finish(self, time_now): class TimerManager(object): def __init__(self): - self._timers = queue.PriorityQueue() + self._queue = [] + self._lock = Lock() def add_timer(self, timer): - self._timers.put_nowait(timer) + with self._lock: + heappush(self._queue, timer) def service_timeouts(self): """ @@ -935,26 +937,29 @@ def service_timeouts(self): :return: next end time, or None """ now = time.time() - while not self._timers.empty(): - timer = self._timers.get_nowait() - try: - if not timer.finish(now): - self._timers.put_nowait(timer) - return timer.end - except Exception: - log.exception("Exception while servicing timeout callback: ") + queue = self._queue + with self._lock: + while queue: + try: + timer = queue[0] + if timer.finish(now): + heappop(queue) + else: + return timer.end + except Exception: + log.exception("Exception while servicing timeout callback: ") @property def next_timeout(self): try: - return self._timers.queue[0].end + return self._queue[0].end except IndexError: pass @property def next_offset(self): try: - next_end = self._timers.queue[0].end + next_end = self._queue[0].end return next_end - time.time() except IndexError: pass From 03068041882c054c27435cd02405ab2072ceb158 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 11:41:47 -0500 Subject: [PATCH 0094/2431] Remove lock in TimerManager --- cassandra/connection.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 707a2a4db0..f3b47c079f 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -925,29 +925,35 @@ class TimerManager(object): def __init__(self): self._queue = [] - self._lock = Lock() + self._new_timers = [] def add_timer(self, timer): - with self._lock: - heappush(self._queue, timer) + """ + called from client thread with a Timer object + """ + self._new_timers.append(timer) def service_timeouts(self): """ run callbacks on all expired timers + Called from the event thread :return: next end time, or None """ - now = time.time() queue = self._queue - with self._lock: - while queue: - try: - timer = queue[0] - if timer.finish(now): - heappop(queue) - else: - return timer.end - except Exception: - log.exception("Exception while servicing timeout callback: ") + new_timers = self._new_timers + while self._new_timers: + t = new_timers.pop() + heappush(queue, t) + now = time.time() + while queue: + try: + timer = queue[0] + if timer.finish(now): + heappop(queue) + else: + return timer.end + except Exception: + log.exception("Exception while servicing timeout callback: ") @property def next_timeout(self): From dd737915a9710b779157c7484a2d9dcc9db3ed32 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 11:53:33 -0500 Subject: [PATCH 0095/2431] Timer queue is now tuple instead of using __lt__ on objects --- cassandra/connection.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index f3b47c079f..0d0c024b0d 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -904,9 +904,6 @@ def __init__(self, timeout, callback): if timeout < 0: self.callback() - def __lt__(self, other): - return self.end < other.end - def cancel(self): self.canceled = True @@ -931,7 +928,7 @@ def add_timer(self, timer): """ called from client thread with a Timer object """ - self._new_timers.append(timer) + self._new_timers.append((timer.end, timer)) def service_timeouts(self): """ @@ -942,12 +939,11 @@ def service_timeouts(self): queue = self._queue new_timers = self._new_timers while self._new_timers: - t = new_timers.pop() - heappush(queue, t) + heappush(queue, new_timers.pop()) now = time.time() while queue: try: - timer = queue[0] + timer = queue[0][1] if timer.finish(now): heappop(queue) else: @@ -958,14 +954,14 @@ def service_timeouts(self): @property def next_timeout(self): try: - return self._queue[0].end + return self._queue[0][0] except IndexError: pass @property def next_offset(self): try: - next_end = self._queue[0].end + next_end = self._queue[0][0] return next_end - time.time() except IndexError: pass From 8f08a885d8c1e4500f914b88c53279238d4251da Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 12:47:17 -0500 Subject: [PATCH 0096/2431] populate ResponseFuture OpterationTimedOut with errors, host --- cassandra/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index b8a236a30e..515f18cbf1 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2609,7 +2609,7 @@ def _cancel_timer(self): self._timer.cancel() def _on_timeout(self): - self._set_final_exception(OperationTimedOut()) + self._set_final_exception(OperationTimedOut(self._errors, self._current_host)) def _make_query_plan(self): # convert the list/generator/etc to an iterator so that subsequent From 7ab74a7f1a7cc03bb3a8cada6fb257d0750bf36b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 13:06:27 -0500 Subject: [PATCH 0097/2431] Remove outdated note about no errback on timeout --- cassandra/cluster.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 515f18cbf1..f5dd46ab0f 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2980,11 +2980,6 @@ def result(self, timeout=_NOT_SET): This is a client-side timeout. For more information about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`. - **Important**: This timeout currently has no effect on callbacks registered - on a :class:`~.ResponseFuture` through :meth:`.ResponseFuture.add_callback` or - :meth:`.ResponseFuture.add_errback`; even if a query exceeds this default - timeout, neither the registered callback or errback will be called. - Example usage:: >>> future = session.execute_async("SELECT * FROM mycf") @@ -3006,6 +3001,9 @@ def result(self, timeout=_NOT_SET): timeout = None self._event.wait(timeout) + # TODO: remove this conditional when deprecated timeout parameter is removed + if not self._event.is_set(): + self._on_timeout() if self._final_result is not _NOT_SET: if self._paging_state is None: return self._final_result From 5a235c46ffad7119e2dd6fff46861ef26d80d0f2 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 14:41:18 -0500 Subject: [PATCH 0098/2431] Make unit test for large DateType work on 32-bit platforms --- tests/unit/test_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 6bfe75b459..bc0eac2da0 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -336,7 +336,7 @@ def test_datetype(self): # beyond 32b expected = 2 ** 33 - self.assertEqual(DateType.deserialize(int64_pack(1000 * expected), 0), datetime.datetime.utcfromtimestamp(expected)) + self.assertEqual(DateType.deserialize(int64_pack(1000 * expected), 0), datetime.datetime(2242, 3, 16, 12, 56, 32)) # less than epoc (PYTHON-119) expected = -770172256 From f4434944b2d4b8045bbb7892195d32ce7d3c2966 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 11 May 2015 14:57:02 -0500 Subject: [PATCH 0099/2431] Convert errors on re-prepare to API exceptions --- cassandra/cluster.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index c5c83a730f..416583aed2 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2875,7 +2875,10 @@ def _execute_after_prepare(self, response): "Got unexpected response when preparing statement " "on host %s: %s" % (self._current_host, response))) elif isinstance(response, ErrorMessage): - self._set_final_exception(response) + if hasattr(response, 'to_exception'): + self._set_final_exception(response.to_exception()) + else: + self._set_final_exception(response) elif isinstance(response, ConnectionException): log.debug("Connection error when preparing statement on host %s: %s", self._current_host, response) From 2464085bccd62fefe1e8e9734be9bb1363da4f00 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 12 May 2015 08:08:17 -0500 Subject: [PATCH 0100/2431] Pass host through to Auth instance in SaslAuthProvider.new_authenticator PYTHON-300 --- cassandra/auth.py | 7 ++++--- .../integration/standard/test_authentication.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cassandra/auth.py b/cassandra/auth.py index 67d302a9e8..0cb4c6b276 100644 --- a/cassandra/auth.py +++ b/cassandra/auth.py @@ -139,8 +139,7 @@ class SaslAuthProvider(AuthProvider): from cassandra.cluster import Cluster from cassandra.auth import SaslAuthProvider - sasl_kwargs = {'host': 'localhost', - 'service': 'dse', + sasl_kwargs = {'service': 'dse', 'mechanism': 'GSSAPI', 'qops': 'auth'.split(',')} auth_provider = SaslAuthProvider(**sasl_kwargs) @@ -152,10 +151,12 @@ class SaslAuthProvider(AuthProvider): def __init__(self, **sasl_kwargs): if SASLClient is None: raise ImportError('The puresasl library has not been installed') + if 'host' in sasl_kwargs: + raise ValueError("kwargs should not contain 'host' since it is passed dynamically to new_authenticator") self.sasl_kwargs = sasl_kwargs def new_authenticator(self, host): - return SaslAuthenticator(**self.sasl_kwargs) + return SaslAuthenticator(host, **self.sasl_kwargs) class SaslAuthenticator(Authenticator): """ diff --git a/tests/integration/standard/test_authentication.py b/tests/integration/standard/test_authentication.py index 3820ef7a90..80c99ede32 100644 --- a/tests/integration/standard/test_authentication.py +++ b/tests/integration/standard/test_authentication.py @@ -138,10 +138,22 @@ def setUp(self): raise unittest.SkipTest('pure-sasl is not installed') def get_authentication_provider(self, username, password): - sasl_kwargs = {'host': 'localhost', - 'service': 'cassandra', + sasl_kwargs = {'service': 'cassandra', 'mechanism': 'PLAIN', 'qops': ['auth'], 'username': username, 'password': password} return SaslAuthProvider(**sasl_kwargs) + + # these could equally be unit tests + def test_host_passthrough(self): + sasl_kwargs = {'service': 'cassandra', + 'mechanism': 'PLAIN'} + provider = SaslAuthProvider(**sasl_kwargs) + host = 'thehostname' + authenticator = provider.new_authenticator(host) + self.assertEqual(authenticator.sasl.host, host) + + def test_host_rejected(self): + sasl_kwargs = {'host': 'something'} + self.assertRaises(ValueError, SaslAuthProvider, **sasl_kwargs) From 3825c3a9120040050c20e7e80b905ebc65bb03d6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 12 May 2015 09:06:22 -0500 Subject: [PATCH 0101/2431] length validation for custom payload map --- cassandra/protocol.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 1a5531022c..b0f9e6e296 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -103,6 +103,8 @@ def update_custom_payload(self, other): if not self.custom_payload: self.custom_payload = {} self.custom_payload.update(other) + if len(self.custom_payload) > 65535: + raise ValueError("Custom payload map exceeds max count allowed by protocol (65535)") def __repr__(self): return '<%s(%s)>' % (self.__class__.__name__, ', '.join('%s=%r' % i for i in _get_params(self))) From 58536f57467db9998417eff57304ef40562f5975 Mon Sep 17 00:00:00 2001 From: Carl Yeksigian Date: Mon, 27 Apr 2015 17:28:47 -0400 Subject: [PATCH 0102/2431] Add trace_id to statement --- cassandra/cluster.py | 2 ++ cassandra/query.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index c5c83a730f..8f41948681 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2700,6 +2700,8 @@ def _set_result(self, response): trace_id = getattr(response, 'trace_id', None) if trace_id: + if self.query: + self.query.trace_id = trace_id self._query_trace = QueryTrace(trace_id, self.session) if isinstance(response, ResultMessage): diff --git a/cassandra/query.py b/cassandra/query.py index 7f7756b3fb..1232514bfa 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -167,6 +167,12 @@ class Statement(object): this will be set to a :class:`.QueryTrace` instance. """ + trace_id = None + """ + If :meth:`.Session.execute()` is run with `trace` set to :const:`True`, + this will be set to the tracing ID from the server. + """ + consistency_level = None """ The :class:`.ConsistencyLevel` to be used for this operation. Defaults From bc3ea9b1bc6ff03811332b7923a3eec3f8abde7d Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 12 May 2015 11:19:51 -0500 Subject: [PATCH 0103/2431] Test for trace_id on query PYTHON-302 --- tests/integration/standard/test_query.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index b30c387b01..6b4eb25e97 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -73,6 +73,21 @@ def test_trace_prints_okay(self): cluster.shutdown() + def test_trace_id_to_query(self): + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + session = cluster.connect() + + query = "SELECT * FROM system.local" + statement = SimpleStatement(query) + self.assertIsNone(statement.trace_id) + future = session.execute_async(statement, trace=True) + + # query should have trace_id, even before trace is obtained + future.result() + self.assertIsNotNone(statement.trace_id) + + cluster.shutdown() + def test_trace_ignores_row_factory(self): cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() From 90e2ace36f56080a77a940fe59a662c89334599b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 12 May 2015 15:04:43 -0500 Subject: [PATCH 0104/2431] cqle: Protect ks/cf identifiers that use keywords PYTHON-244 --- cassandra/cqlengine/management.py | 15 +++---- cassandra/cqlengine/models.py | 43 +++++++++++-------- .../integration/cqlengine/model/test_model.py | 37 +++++++++++++++- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/cassandra/cqlengine/management.py b/cassandra/cqlengine/management.py index 7ea642af8d..d84406f959 100644 --- a/cassandra/cqlengine/management.py +++ b/cassandra/cqlengine/management.py @@ -81,7 +81,7 @@ def create_keyspace(name, strategy_class, replication_factor, durable_writes=Tru query = """ CREATE KEYSPACE {} WITH REPLICATION = {} - """.format(name, json.dumps(replication_map).replace('"', "'")) + """.format(metadata.protect_name(name), json.dumps(replication_map).replace('"', "'")) if strategy_class != 'SimpleStrategy': query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false') @@ -163,7 +163,7 @@ def drop_keyspace(name): cluster = get_cluster() if name in cluster.metadata.keyspaces: - execute("DROP KEYSPACE {}".format(name)) + execute("DROP KEYSPACE {}".format(metadata.protect_name(name))) def sync_table(model): @@ -191,9 +191,8 @@ def sync_table(model): if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") - # construct query string cf_name = model.column_family_name() - raw_cf_name = model.column_family_name(include_keyspace=False) + raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() @@ -433,7 +432,7 @@ def setter(key, limited_to_strategy=None): def get_fields(model): # returns all fields that aren't part of the PK ks_name = model._get_keyspace() - col_family = model.column_family_name(include_keyspace=False) + col_family = model._raw_column_family_name() field_types = ['regular', 'static'] query = "select * from system.schema_columns where keyspace_name = %s and columnfamily_name = %s" tmp = execute(query, [ks_name, col_family]) @@ -452,7 +451,7 @@ def get_table_settings(model): # returns the table as provided by the native driver for a given model cluster = get_cluster() ks = model._get_keyspace() - table = model.column_family_name(include_keyspace=False) + table = model._raw_column_family_name() table = cluster.metadata.keyspaces[ks].tables[table] return table @@ -520,11 +519,11 @@ def drop_table(model): meta = get_cluster().metadata ks_name = model._get_keyspace() - raw_cf_name = model.column_family_name(include_keyspace=False) + raw_cf_name = model._raw_column_family_name() try: meta.keyspaces[ks_name].tables[raw_cf_name] - execute('drop table {};'.format(model.column_family_name(include_keyspace=True))) + execute('drop table {};'.format(model.column_family_name())) except KeyError: pass diff --git a/cassandra/cqlengine/models.py b/cassandra/cqlengine/models.py index 0baa0906a0..4f6d120250 100644 --- a/cassandra/cqlengine/models.py +++ b/cassandra/cqlengine/models.py @@ -23,6 +23,7 @@ from cassandra.cqlengine import query from cassandra.cqlengine.query import DoesNotExist as _DoesNotExist from cassandra.cqlengine.query import MultipleObjectsReturned as _MultipleObjectsReturned +from cassandra.metadata import protect_name from cassandra.util import OrderedDict log = logging.getLogger(__name__) @@ -353,6 +354,8 @@ class MultipleObjectsReturned(_MultipleObjectsReturned): _if_not_exists = False # optional if_not_exists flag to check existence before insertion + _table_name = None # used internally to cache a derived table name + def __init__(self, **values): self._values = {} self._ttl = self.__default_ttl__ @@ -504,27 +507,33 @@ def column_family_name(cls, include_keyspace=True): Returns the column family name if it's been defined otherwise, it creates it from the module and class name """ - cf_name = '' - if cls.__table_name__: - cf_name = cls.__table_name__.lower() - else: - # get polymorphic base table names if model is polymorphic - if cls._is_polymorphic and not cls._is_polymorphic_base: - return cls._polymorphic_base.column_family_name(include_keyspace=include_keyspace) + cf_name = protect_name(cls._raw_column_family_name()) + if include_keyspace: + return '{}.{}'.format(protect_name(cls._get_keyspace()), cf_name) + + return cf_name - camelcase = re.compile(r'([a-z])([A-Z])') - ccase = lambda s: camelcase.sub(lambda v: '{}_{}'.format(v.group(1), v.group(2).lower()), s) - cf_name += ccase(cls.__name__) - # trim to less than 48 characters or cassandra will complain - cf_name = cf_name[-48:] - cf_name = cf_name.lower() - cf_name = re.sub(r'^_+', '', cf_name) + @classmethod + def _raw_column_family_name(cls): + if not cls._table_name: + if cls.__table_name__: + cls._table_name = cls.__table_name__.lower() + else: + if cls._is_polymorphic and not cls._is_polymorphic_base: + cls._table_name = cls._polymorphic_base._raw_column_family_name() + else: + camelcase = re.compile(r'([a-z])([A-Z])') + ccase = lambda s: camelcase.sub(lambda v: '{}_{}'.format(v.group(1), v.group(2).lower()), s) - if not include_keyspace: - return cf_name + cf_name = ccase(cls.__name__) + # trim to less than 48 characters or cassandra will complain + cf_name = cf_name[-48:] + cf_name = cf_name.lower() + cf_name = re.sub(r'^_+', '', cf_name) + cls._table_name = cf_name - return '{}.{}'.format(cls._get_keyspace(), cf_name) + return cls._table_name def validate(self): """ diff --git a/tests/integration/cqlengine/model/test_model.py b/tests/integration/cqlengine/model/test_model.py index bdb045954a..c37e088f54 100644 --- a/tests/integration/cqlengine/model/test_model.py +++ b/tests/integration/cqlengine/model/test_model.py @@ -14,8 +14,9 @@ from unittest import TestCase -from cassandra.cqlengine.models import Model, ModelDefinitionException from cassandra.cqlengine import columns +from cassandra.cqlengine.management import sync_table, drop_table, create_keyspace_simple, drop_keyspace +from cassandra.cqlengine.models import Model, ModelDefinitionException class TestModel(TestCase): @@ -49,6 +50,40 @@ class EqualityModel1(Model): self.assertEqual(m0, m0) self.assertNotEqual(m0, m1) + def test_keywords_as_names(self): + create_keyspace_simple('keyspace', 1) + + class table(Model): + __keyspace__ = 'keyspace' + select = columns.Integer(primary_key=True) + table = columns.Text() + + # create should work + drop_table(table) + sync_table(table) + + created = table.create(select=0, table='table') + selected = table.objects(select=0)[0] + self.assertEqual(created.select, selected.select) + self.assertEqual(created.table, selected.table) + + # alter should work + class table(Model): + __keyspace__ = 'keyspace' + select = columns.Integer(primary_key=True) + table = columns.Text() + where = columns.Text() + + sync_table(table) + + created = table.create(select=1, table='table') + selected = table.objects(select=1)[0] + self.assertEqual(created.select, selected.select) + self.assertEqual(created.table, selected.table) + self.assertEqual(created.where, selected.where) + + drop_keyspace('keyspace') + class BuiltInAttributeConflictTest(TestCase): """tests Model definitions that conflict with built-in attributes/methods""" From 5c19ad507ceba7731494ddbe8fff06d146e8c789 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Thu, 7 May 2015 16:42:42 -0500 Subject: [PATCH 0105/2431] Fixing timeout issues --- tests/integration/long/test_large_data.py | 86 +++++++++++++++++++---- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/tests/integration/long/test_large_data.py b/tests/integration/long/test_large_data.py index acb203568b..0b77feb0cb 100644 --- a/tests/integration/long/test_large_data.py +++ b/tests/integration/long/test_large_data.py @@ -72,6 +72,7 @@ def make_session_and_keyspace(self): def batch_futures(self, session, statement_generator): concurrency = 10 futures = Queue(maxsize=concurrency) + number_of_timeouts = 0 for i, statement in enumerate(statement_generator): if i > 0 and i % (concurrency - 1) == 0: # clear the existing queue @@ -80,6 +81,7 @@ def batch_futures(self, session, statement_generator): futures.get_nowait().result() except (OperationTimedOut, WriteTimeout): ex_type, ex, tb = sys.exc_info() + number_of_timeouts += 1 log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb time.sleep(1) @@ -94,11 +96,13 @@ def batch_futures(self, session, statement_generator): futures.get_nowait().result() except (OperationTimedOut, WriteTimeout): ex_type, ex, tb = sys.exc_info() + number_of_timeouts += 1 log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb time.sleep(1) except Empty: break + return number_of_timeouts def test_wide_rows(self): table = 'wide_rows' @@ -120,48 +124,103 @@ def test_wide_rows(self): session.cluster.shutdown() def test_wide_batch_rows(self): + """ + Test for inserting wide rows with batching + + test_wide_batch_rows tests inserting a wide row of data using batching. It will then attempt to query + that data and ensure that all of it has been inserted appropriately. + + @expected_result all items should be inserted, and verified. + + @test_category queries:batch + """ + + # Table Creation table = 'wide_batch_rows' session = self.make_session_and_keyspace() session.execute('CREATE TABLE %s (k INT, i INT, PRIMARY KEY(k, i))' % table) - # Write + # Run batch insert statement = 'BEGIN BATCH ' - for i in range(2000): + to_insert = 2000 + for i in range(to_insert): statement += 'INSERT INTO %s (k, i) VALUES (%s, %s) ' % (table, 0, i) statement += 'APPLY BATCH' statement = SimpleStatement(statement, consistency_level=ConsistencyLevel.QUORUM) - session.execute(statement) - # Read - results = session.execute('SELECT i FROM %s WHERE k=%s' % (table, 0)) + # Execute insert with larger timeout, since it's a wide row + try: + session.execute(statement,timeout=30.0) + + except OperationTimedOut: + #If we timeout on insertion that's bad but it could be just slow underlying c* + #Attempt to validate anyway, we will fail if we don't get the right data back. + ex_type, ex, tb = sys.exc_info() + log.warn("Batch wide row insertion timed out, this may require additional investigation") + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb # Verify - for i, row in enumerate(results): - self.assertEqual(row['i'], i) + results = session.execute('SELECT i FROM %s WHERE k=%s' % (table, 0)) + lastvalue = 0 + for j, row in enumerate(results): + lastValue=row['i'] + self.assertEqual(lastValue, j) + + #check the last value make sure it's what we expect + index_value = to_insert-1 + self.assertEqual(lastValue,index_value,"Verification failed only found {0} inserted we were expecting {1}".format(j,index_value)) session.cluster.shutdown() def test_wide_byte_rows(self): + """ + Test for inserting wide row of bytes + + test_wide_batch_rows tests inserting a wide row of data bytes. It will then attempt to query + that data and ensure that all of it has been inserted appropriately. + + @expected_result all items should be inserted, and verified. + + @test_category queries + """ + + # Table creation table = 'wide_byte_rows' session = self.make_session_and_keyspace() session.execute('CREATE TABLE %s (k INT, i INT, v BLOB, PRIMARY KEY(k, i))' % table) + # Prepare statement and run insertions + to_insert = 100000 prepared = session.prepare('INSERT INTO %s (k, i, v) VALUES (0, ?, 0xCAFE)' % (table, )) - - # Write - self.batch_futures(session, (prepared.bind((i, )) for i in range(100000))) + timeouts = self.batch_futures(session, (prepared.bind((i, )) for i in range(to_insert))) # Read results = session.execute('SELECT i, v FROM %s WHERE k=0' % (table, )) + # number of expected results + expected_results = to_insert-timeouts-1 + # Verify bb = pack('>H', 0xCAFE) - for row in results: + for i, row in enumerate(results): self.assertEqual(row['v'], bb) + self.assertGreaterEqual(i, expected_results, "Verification failed only found {0} inserted we were expecting {1}".format(i,expected_results)) + session.cluster.shutdown() def test_large_text(self): + """ + Test for inserting a large text field + + test_large_text tests inserting a large text field into a row. + + @expected_result the large text value should be inserted. When the row is queried it should match the original + value that was inserted + + @test_category queries + """ table = 'large_text' session = self.make_session_and_keyspace() session.execute('CREATE TABLE %s (k int PRIMARY KEY, txt text)' % table) @@ -178,8 +237,11 @@ def test_large_text(self): result = session.execute('SELECT * FROM %s WHERE k=%s' % (table, 0)) # Verify - for row in result: + found_result = False + for i, row in enumerate(result): self.assertEqual(row['txt'], text) + found_result = True + self.assertTrue(found_result, "No results were found") session.cluster.shutdown() From c841c412995788c8ea6e5c96365839e6a28acf01 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 13 May 2015 14:59:10 -0500 Subject: [PATCH 0106/2431] cqle: Make sure user type is registered on sync even if no change is made. --- cassandra/cqlengine/management.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cassandra/cqlengine/management.py b/cassandra/cqlengine/management.py index 7ea642af8d..284c1404fb 100644 --- a/cassandra/cqlengine/management.py +++ b/cassandra/cqlengine/management.py @@ -313,6 +313,8 @@ def _sync_type(ks_name, type_model, omit_subtypes=None): if field.db_field_name not in defined_fields: execute("ALTER TYPE {} ADD {}".format(type_name_qualified, field.get_column_def())) + type_model.register_for_keyspace(ks_name) + if len(defined_fields) == len(model_fields): log.info("Type %s did not require synchronization", type_name_qualified) return @@ -321,8 +323,6 @@ def _sync_type(ks_name, type_model, omit_subtypes=None): if db_fields_not_in_model: log.info("Type %s has fields not referenced by model: %s", type_name_qualified, db_fields_not_in_model) - type_model.register_for_keyspace(ks_name) - def get_create_type(type_model, keyspace): type_meta = metadata.UserType(keyspace, From d9cbb6e2689a31ffcb6cd5d243d2945c6233b697 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 13 May 2015 14:59:53 -0500 Subject: [PATCH 0107/2431] cqle: Correct CQL encoding for collections with nested types. PYTHON-311 Removes superfluous quoting mechanism, allowing collections to correctly encode contents (instead of being implicitly strinified). --- cassandra/cqlengine/columns.py | 68 +------------------ cassandra/cqlengine/statements.py | 2 +- .../columns/test_container_columns.py | 12 ++-- .../cqlengine/columns/test_value_io.py | 9 --- .../statements/test_update_statement.py | 10 +-- 5 files changed, 15 insertions(+), 86 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index ae17779590..8a09ba1254 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -73,25 +73,6 @@ def get_property(self): return property(_get, _set) -class ValueQuoter(object): - """ - contains a single value, which will quote itself for CQL insertion statements - """ - def __init__(self, value): - self.value = value - - def __str__(self): - raise NotImplementedError - - def __repr__(self): - return self.__str__() - - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.value == other.value - return False - - class Column(object): # the cassandra type this column maps to @@ -701,24 +682,12 @@ def sub_columns(self): return [self.value_col] -class BaseContainerQuoter(ValueQuoter): - - def __nonzero__(self): - return bool(self.value) - - class Set(BaseContainerColumn): """ Stores a set of unordered, unique values http://www.datastax.com/documentation/cql/3.1/cql/cql_using/use_set_t.html """ - class Quoter(BaseContainerQuoter): - - def __str__(self): - cq = cql_quote - return '{' + ', '.join([cq(v) for v in self.value]) + '}' - def __init__(self, value_type, strict=True, default=set, **kwargs): """ :param value_type: a column class indicating the types of the value @@ -753,10 +722,7 @@ def to_python(self, value): def to_database(self, value): if value is None: return None - - if isinstance(value, self.Quoter): - return value - return self.Quoter({self.value_col.to_database(v) for v in value}) + return {self.value_col.to_database(v) for v in value} class List(BaseContainerColumn): @@ -765,15 +731,6 @@ class List(BaseContainerColumn): http://www.datastax.com/documentation/cql/3.1/cql/cql_using/use_list_t.html """ - class Quoter(BaseContainerQuoter): - - def __str__(self): - cq = cql_quote - return '[' + ', '.join([cq(v) for v in self.value]) + ']' - - def __nonzero__(self): - return bool(self.value) - def __init__(self, value_type, default=list, **kwargs): """ :param value_type: a column class indicating the types of the value @@ -799,9 +756,7 @@ def to_python(self, value): def to_database(self, value): if value is None: return None - if isinstance(value, self.Quoter): - return value - return self.Quoter([self.value_col.to_database(v) for v in value]) + return [self.value_col.to_database(v) for v in value] class Map(BaseContainerColumn): @@ -810,21 +765,6 @@ class Map(BaseContainerColumn): http://www.datastax.com/documentation/cql/3.1/cql/cql_using/use_map_t.html """ - class Quoter(BaseContainerQuoter): - - def __str__(self): - cq = cql_quote - return '{' + ', '.join([cq(k) + ':' + cq(v) for k, v in self.value.items()]) + '}' - - def get(self, key): - return self.value.get(key) - - def keys(self): - return self.value.keys() - - def items(self): - return self.value.items() - def __init__(self, key_type, value_type, default=dict, **kwargs): """ :param key_type: a column class indicating the types of the key @@ -866,9 +806,7 @@ def to_python(self, value): def to_database(self, value): if value is None: return None - if isinstance(value, self.Quoter): - return value - return self.Quoter({self.key_col.to_database(k): self.value_col.to_database(v) for k, v in value.items()}) + return {self.key_col.to_database(k): self.value_col.to_database(v) for k, v in value.items()} @property def sub_columns(self): diff --git a/cassandra/cqlengine/statements.py b/cassandra/cqlengine/statements.py index f64cc7a0de..866f6f2958 100644 --- a/cassandra/cqlengine/statements.py +++ b/cassandra/cqlengine/statements.py @@ -658,7 +658,7 @@ def get_context(self): class InsertStatement(AssignmentStatement): - """ an cql insert select statement """ + """ an cql insert statement """ def __init__(self, table, diff --git a/tests/integration/cqlengine/columns/test_container_columns.py b/tests/integration/cqlengine/columns/test_container_columns.py index 03a94e0df4..a9c8b35965 100644 --- a/tests/integration/cqlengine/columns/test_container_columns.py +++ b/tests/integration/cqlengine/columns/test_container_columns.py @@ -161,8 +161,8 @@ def test_to_python(self): column = columns.Set(JsonTestColumn) val = {1, 2, 3} db_val = column.to_database(val) - assert db_val.value == {json.dumps(v) for v in val} - py_val = column.to_python(db_val.value) + assert db_val == {json.dumps(v) for v in val} + py_val = column.to_python(db_val) assert py_val == val def test_default_empty_container_saving(self): @@ -277,8 +277,8 @@ def test_to_python(self): column = columns.List(JsonTestColumn) val = [1, 2, 3] db_val = column.to_database(val) - assert db_val.value == [json.dumps(v) for v in val] - py_val = column.to_python(db_val.value) + assert db_val == [json.dumps(v) for v in val] + py_val = column.to_python(db_val) assert py_val == val def test_default_empty_container_saving(self): @@ -495,8 +495,8 @@ def test_to_python(self): column = columns.Map(JsonTestColumn, JsonTestColumn) val = {1: 2, 3: 4, 5: 6} db_val = column.to_database(val) - assert db_val.value == {json.dumps(k):json.dumps(v) for k,v in val.items()} - py_val = column.to_python(db_val.value) + assert db_val == {json.dumps(k):json.dumps(v) for k,v in val.items()} + py_val = column.to_python(db_val) assert py_val == val def test_default_empty_container_saving(self): diff --git a/tests/integration/cqlengine/columns/test_value_io.py b/tests/integration/cqlengine/columns/test_value_io.py index 04be247a84..17f96e2242 100644 --- a/tests/integration/cqlengine/columns/test_value_io.py +++ b/tests/integration/cqlengine/columns/test_value_io.py @@ -22,7 +22,6 @@ from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.management import drop_table from cassandra.cqlengine.models import Model -from cassandra.cqlengine.columns import ValueQuoter from cassandra.cqlengine import columns import unittest @@ -211,11 +210,3 @@ class TestDecimalIO(BaseColumnIOTest): def comparator_converter(self, val): return Decimal(val) - -class TestQuoter(unittest.TestCase): - - def test_equals(self): - assert ValueQuoter(False) == ValueQuoter(False) - assert ValueQuoter(1) == ValueQuoter(1) - assert ValueQuoter("foo") == ValueQuoter("foo") - assert ValueQuoter(1.55) == ValueQuoter(1.55) diff --git a/tests/integration/cqlengine/statements/test_update_statement.py b/tests/integration/cqlengine/statements/test_update_statement.py index 070710146b..0f77d53ca4 100644 --- a/tests/integration/cqlengine/statements/test_update_statement.py +++ b/tests/integration/cqlengine/statements/test_update_statement.py @@ -60,25 +60,25 @@ def test_additional_rendering(self): def test_update_set_add(self): us = UpdateStatement('table') - us.add_assignment_clause(SetUpdateClause('a', Set.Quoter({1}), operation='add')) + us.add_assignment_clause(SetUpdateClause('a', {1}, operation='add')) self.assertEqual(six.text_type(us), 'UPDATE table SET "a" = "a" + %(0)s') def test_update_empty_set_add_does_not_assign(self): us = UpdateStatement('table') - us.add_assignment_clause(SetUpdateClause('a', Set.Quoter(set()), operation='add')) + us.add_assignment_clause(SetUpdateClause('a', set(), operation='add')) self.assertEqual(six.text_type(us), 'UPDATE table SET "a" = "a" + %(0)s') def test_update_empty_set_removal_does_not_assign(self): us = UpdateStatement('table') - us.add_assignment_clause(SetUpdateClause('a', Set.Quoter(set()), operation='remove')) + us.add_assignment_clause(SetUpdateClause('a', set(), operation='remove')) self.assertEqual(six.text_type(us), 'UPDATE table SET "a" = "a" - %(0)s') def test_update_list_prepend_with_empty_list(self): us = UpdateStatement('table') - us.add_assignment_clause(ListUpdateClause('a', List.Quoter([]), operation='prepend')) + us.add_assignment_clause(ListUpdateClause('a', [], operation='prepend')) self.assertEqual(six.text_type(us), 'UPDATE table SET "a" = %(0)s + "a"') def test_update_list_append_with_empty_list(self): us = UpdateStatement('table') - us.add_assignment_clause(ListUpdateClause('a', List.Quoter([]), operation='append')) + us.add_assignment_clause(ListUpdateClause('a', [], operation='append')) self.assertEqual(six.text_type(us), 'UPDATE table SET "a" = "a" + %(0)s') From 28359d5ec873ac24388d99f031aa5734dfdbad4d Mon Sep 17 00:00:00 2001 From: GregBestland Date: Wed, 13 May 2015 15:49:35 -0500 Subject: [PATCH 0108/2431] potential fix to travis unit test errors --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6c4aa31a27..d694e7ab03 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,8 @@ deps = nose [testenv] deps = {[base]deps} + sure==1.2.3 blist - sure setenv = USE_CASS_EXTERNAL=1 commands = {envpython} setup.py build_ext --inplace nosetests --verbosity=2 tests/unit/ From 45a09d36f979f031e8132572f6bdaa929d3124f7 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 13 May 2015 15:56:47 -0500 Subject: [PATCH 0109/2431] Synchronize reconnection_handler cancelation in CC shutdown Fixes a possible race where ControlConnection._reconnection_handler could be removed during shutdown. --- cassandra/cluster.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 436bfd929e..87628c1008 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2024,6 +2024,11 @@ def _submit(self, *args, **kwargs): return None def shutdown(self): + # stop trying to reconnect (if we are) + with self._reconnection_lock: + if self._reconnection_handler: + self._reconnection_handler.cancel() + with self._lock: if self._is_shutdown: return @@ -2031,10 +2036,6 @@ def shutdown(self): self._is_shutdown = True log.debug("Shutting down control connection") - # stop trying to reconnect (if we are) - if self._reconnection_handler: - self._reconnection_handler.cancel() - if self._connection: self._connection.close() del self._connection From e9b948b87577d3882add31d6a82b9ce468670617 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Wed, 13 May 2015 16:34:33 -0700 Subject: [PATCH 0110/2431] fixed typo in test harness detecting C* versions --- tests/integration/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 4b3c538145..d47d1d2002 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -119,9 +119,9 @@ def _tuple_version(version_string): log.info('Using Cassandra version: %s', CASSANDRA_VERSION) CCM_KWARGS['version'] = CASSANDRA_VERSION -if CASSANDRA_VERSION > '2.1': +if CASSANDRA_VERSION >= '2.1': default_protocol_version = 3 -elif CASSANDRA_VERSION > '2.0': +elif CASSANDRA_VERSION >= '2.0': default_protocol_version = 2 else: default_protocol_version = 1 From cef495e8563a36152cf23e3b71646d2b7f15b121 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Wed, 13 May 2015 16:53:44 -0700 Subject: [PATCH 0111/2431] fixed typo in test harness detecting C* 3.0 version --- tests/integration/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3ea397f01e..8933eefd48 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -118,11 +118,11 @@ def _tuple_version(version_string): log.info('Using Cassandra version: %s', CASSANDRA_VERSION) CCM_KWARGS['version'] = CASSANDRA_VERSION -if CASSANDRA_VERSION > '3.0': +if CASSANDRA_VERSION >= '3.0': default_protocol_version = 4 -elif CASSANDRA_VERSION > '2.1': +elif CASSANDRA_VERSION >= '2.1': default_protocol_version = 3 -elif CASSANDRA_VERSION > '2.0': +elif CASSANDRA_VERSION >= '2.0': default_protocol_version = 2 else: default_protocol_version = 1 From 536300ddb9973a1378a2f7b9c6b304dfe5f1c2f9 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 13 May 2015 20:02:48 -0500 Subject: [PATCH 0112/2431] For DefaultConnection, default to gevent only if actually patched. PYTHON-244 Fixes an issue where gevent is chosen based on module loaded, not monkey patching. In some debuggers, (PyCharm, for example), this would cause connecting to timeout because gevent reactor is used without monkey patching. --- cassandra/cluster.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 56e34e97dd..eca637d0e3 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -79,10 +79,16 @@ def _is_eventlet_monkey_patched(): import eventlet.patcher return eventlet.patcher.is_monkey_patched('socket') +def _is_gevent_monkey_patched(): + if 'gevent.monkey' not in sys.modules: + return False + import gevent.socket + return socket.socket is gevent.socket.socket + # default to gevent when we are monkey patched with gevent, eventlet when # monkey patched with eventlet, otherwise if libev is available, use that as # the default because it's fastest. Otherwise, use asyncore. -if 'gevent.monkey' in sys.modules: +if _is_gevent_monkey_patched(): from cassandra.io.geventreactor import GeventConnection as DefaultConnection elif _is_eventlet_monkey_patched(): from cassandra.io.eventletreactor import EventletConnection as DefaultConnection From c4bfe375bab678210cd3826c3eb7e471b81a79c1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 14 May 2015 10:25:08 -0500 Subject: [PATCH 0113/2431] Warn when UDT registered, but protocol_version < 3 PYTHON-305 --- cassandra/cluster.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 56e34e97dd..0c640988eb 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -624,6 +624,11 @@ def __init__(self, street, zipcode): print row.id, row.location.street, row.location.zipcode """ + if self.protocol_version < 3: + log.warning("User Type serialization is only supported in native protocol version 3+ (%d in use). " + "CQL encoding for simple statements will still work, but named tuples will " + "be returned when reading type %s.%s.", self.protocol_version, keyspace, user_type) + self._user_types[keyspace][user_type] = klass for session in self.sessions: session.user_type_registered(keyspace, user_type, klass) From 7718544a626c7188725c1738f80ed06e48cbdb31 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 14 May 2015 11:05:18 -0500 Subject: [PATCH 0114/2431] Deprecate Cluster.refresh_schema and submit_refresh_schema PYTHON-291 Use Cluster.refresh_*_metadata instead --- cassandra/cluster.py | 57 ++++++++++++++++++++++++++++++++++ docs/api/cassandra/cluster.rst | 8 +++++ 2 files changed, 65 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 56e34e97dd..c48a1ebffd 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1131,6 +1131,9 @@ def _ensure_core_connections(self): def refresh_schema(self, keyspace=None, table=None, usertype=None, max_schema_agreement_wait=None): """ + .. deprecated:: 2.6.0 + Use refresh_*_metadata instead + Synchronously refresh the schema metadata. By default, the timeout for this operation is governed by :attr:`~.Cluster.max_schema_agreement_wait` @@ -1142,18 +1145,72 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, max_schema_ag An Exception is raised if schema refresh fails for any reason. """ + msg = "refresh_schema is deprecated. Use Cluster.refresh_*_metadata instead." + warnings.warn(msg, DeprecationWarning) + log.warning(msg) if not self.control_connection.refresh_schema(keyspace, table, usertype, max_schema_agreement_wait): raise Exception("Schema was not refreshed. See log for details.") def submit_schema_refresh(self, keyspace=None, table=None, usertype=None): """ + .. deprecated:: 2.6.0 + Use refresh_*_metadata instead + Schedule a refresh of the internal representation of the current schema for this cluster. If `keyspace` is specified, only that keyspace will be refreshed, and likewise for `table`. """ + msg = "submit_schema_refresh is deprecated. Use Cluster.refresh_*_metadata instead." + warnings.warn(msg, DeprecationWarning) + log.warning(msg) return self.executor.submit( self.control_connection.refresh_schema, keyspace, table, usertype) + def refresh_schema_metadata(self, max_schema_agreement_wait=None): + """ + Synchronously refresh all schema metadata. + + By default, the timeout for this operation is governed by :attr:`~.Cluster.max_schema_agreement_wait` + and :attr:`~.Cluster.control_connection_timeout`. + + Passing max_schema_agreement_wait here overrides :attr:`~.Cluster.max_schema_agreement_wait`. + + Setting max_schema_agreement_wait <= 0 will bypass schema agreement and refresh schema immediately. + + An Exception is raised if schema refresh fails for any reason. + """ + if not self.control_connection.refresh_schema(schema_agreement_wait=max_schema_agreement_wait): + raise Exception("Schema metadata was not refreshed. See log for details.") + + def refresh_keyspace_metadata(self, keyspace, max_schema_agreement_wait=None): + """ + Synchronously refresh keyspace metadata. This applies to keyspace-level information such as replication + and durability settings. It does not refresh tables, types, etc. contained in the keyspace. + + See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior + """ + if not self.control_connection.refresh_schema(keyspace, schema_agreement_wait=max_schema_agreement_wait): + raise Exception("Keyspace metadata was not refreshed. See log for details.") + + def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None): + """ + Synchronously refresh table metadata. This applies to a table, and any triggers or indexes attached + to the table. + + See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior + """ + if not self.control_connection.refresh_schema(keyspace, table, schema_agreement_wait=max_schema_agreement_wait): + raise Exception("Table metadata was not refreshed. See log for details.") + + def refresh_type_metadata(self, keyspace, user_type, max_schema_agreement_wait=None): + """ + Synchronously refresh user defined type metadata. + + See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior + """ + if not self.control_connection.refresh_schema(keyspace, usertype=user_type, schema_agreement_wait=max_schema_agreement_wait): + raise Exception("User Type metadata was not refreshed. See log for details.") + def refresh_nodes(self): """ Synchronously refresh the node list and token metadata diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 6532c9a026..886d07160e 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -65,6 +65,14 @@ .. automethod:: set_max_connections_per_host + .. automethod:: refresh_schema_metadata + + .. automethod:: refresh_keyspace_metadata + + .. automethod:: refresh_table_metadata + + .. automethod:: refresh_type_metadata + .. automethod:: refresh_schema .. automethod:: refresh_nodes From 12bb183d367bd2d96b8e52639246effefa5712d2 Mon Sep 17 00:00:00 2001 From: blerer Date: Tue, 12 May 2015 10:18:09 +0200 Subject: [PATCH 0115/2431] Add support for byte and smallint types --- cassandra/cqltypes.py | 25 ++++++++++++++++++++++++- cassandra/protocol.py | 4 +++- tests/integration/datatype_utils.py | 8 ++++++++ tests/unit/test_marshalling.py | 6 +++++- tests/unit/test_types.py | 6 +++++- 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 5b95d4c04b..48cbc198c1 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -44,7 +44,7 @@ import warnings -from cassandra.marshal import (int8_pack, int8_unpack, +from cassandra.marshal import (int8_pack, int8_unpack, int16_pack, int16_unpack, uint16_pack, uint16_unpack, uint32_pack, uint32_unpack, int32_pack, int32_unpack, int64_pack, int64_unpack, float_pack, float_unpack, double_pack, double_unpack, @@ -424,6 +424,17 @@ def deserialize(byts, protocol_version): def serialize(truth, protocol_version): return int8_pack(truth) +class TinyIntType(_CassandraType): + typename = 'tinyint' + + @staticmethod + def deserialize(byts, protocol_version): + return int8_unpack(byts) + + @staticmethod + def serialize(byts, protocol_version): + return int8_pack(byts) + if six.PY2: class AsciiType(_CassandraType): @@ -650,6 +661,18 @@ def deserialize(byts, protocol_version): return util.Date(days) +class SmallIntType(_CassandraType): + typename = 'smallint' + + @staticmethod + def deserialize(byts, protocol_version): + return int16_unpack(byts) + + @staticmethod + def serialize(byts, protocol_version): + return int16_pack(byts) + + class TimeType(_CassandraType): typename = 'time' diff --git a/cassandra/protocol.py b/cassandra/protocol.py index bd929dad52..031501bee0 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -34,7 +34,7 @@ LongType, MapType, SetType, TimeUUIDType, UTF8Type, UUIDType, UserType, TupleType, lookup_casstype, SimpleDateType, - TimeType) + TimeType, TinyIntType, SmallIntType) from cassandra.policies import WriteType log = logging.getLogger(__name__) @@ -533,6 +533,8 @@ class ResultMessage(_MessageType): 0x0010: InetAddressType, 0x0011: SimpleDateType, 0x0012: TimeType, + 0x0013: SmallIntType, + 0x0014: TinyIntType, 0x0020: ListType, 0x0021: MapType, 0x0022: SetType, diff --git a/tests/integration/datatype_utils.py b/tests/integration/datatype_utils.py index fb437d7539..a7d8aeb1f8 100644 --- a/tests/integration/datatype_utils.py +++ b/tests/integration/datatype_utils.py @@ -60,6 +60,8 @@ def update_datatypes(): if _cass_version >= (3, 0, 0): PRIMITIVE_DATATYPES.append('date') PRIMITIVE_DATATYPES.append('time') + PRIMITIVE_DATATYPES.append('tinyint') + PRIMITIVE_DATATYPES.append('smallint') def get_sample_data(): @@ -117,6 +119,12 @@ def get_sample_data(): elif datatype == 'time': sample_data[datatype] = time(16, 47, 25, 7) + elif datatype == 'tinyint': + sample_data[datatype] = 123 + + elif datatype == 'smallint': + sample_data[datatype] = 32523 + else: raise Exception("Missing handling of {0}".format(datatype)) diff --git a/tests/unit/test_marshalling.py b/tests/unit/test_marshalling.py index bb9a406882..eeefb0f174 100644 --- a/tests/unit/test_marshalling.py +++ b/tests/unit/test_marshalling.py @@ -81,7 +81,11 @@ (b'\x00\x01\x00\x10\xafYC\xa3\xea<\x11\xe1\xabc\xc4,\x03"y\xf0', 'ListType(TimeUUIDType)', [UUID(bytes=b'\xafYC\xa3\xea<\x11\xe1\xabc\xc4,\x03"y\xf0')]), (b'\x80\x00\x00\x01', 'SimpleDateType', Date(1)), (b'\x7f\xff\xff\xff', 'SimpleDateType', Date('1969-12-31')), - (b'\x00\x00\x00\x00\x00\x00\x00\x01', 'TimeType', Time(1)) + (b'\x00\x00\x00\x00\x00\x00\x00\x01', 'TimeType', Time(1)), + (b'\x7f', 'TinyIntType', 127), + (b'\xff\xff\xff\x80', 'TinyIntType', -128), + (b'\xff\xff\x80\x00', 'SmallIntType', 32767), + (b'\xff\xff\x80\x00', 'SmallIntType', -32768) ) ordered_map_value = OrderedMapSerializedKey(UTF8Type, 2) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index bc0eac2da0..4386380359 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -26,7 +26,7 @@ from cassandra.cqltypes import (BooleanType, lookup_casstype_simple, lookup_casstype, LongType, DecimalType, SetType, cql_typename, CassandraType, UTF8Type, parse_casstype_args, - SimpleDateType, TimeType, + SimpleDateType, TimeType, TinyIntType, SmallIntType, EmptyValue, _CassandraType, DateType, int64_pack) from cassandra.encoder import cql_quote from cassandra.protocol import (write_string, read_longstring, write_stringmap, @@ -55,6 +55,8 @@ def test_lookup_casstype_simple(self): self.assertEqual(lookup_casstype_simple('UTF8Type'), cassandra.cqltypes.UTF8Type) self.assertEqual(lookup_casstype_simple('DateType'), cassandra.cqltypes.DateType) self.assertEqual(lookup_casstype_simple('SimpleDateType'), cassandra.cqltypes.SimpleDateType) + self.assertEqual(lookup_casstype_simple('TinyIntType'), cassandra.cqltypes.TinyIntType) + self.assertEqual(lookup_casstype_simple('SmallIntType'), cassandra.cqltypes.SmallIntType) self.assertEqual(lookup_casstype_simple('TimeUUIDType'), cassandra.cqltypes.TimeUUIDType) self.assertEqual(lookup_casstype_simple('TimeType'), cassandra.cqltypes.TimeType) self.assertEqual(lookup_casstype_simple('UUIDType'), cassandra.cqltypes.UUIDType) @@ -87,6 +89,8 @@ def test_lookup_casstype(self): self.assertEqual(lookup_casstype('UTF8Type'), cassandra.cqltypes.UTF8Type) self.assertEqual(lookup_casstype('DateType'), cassandra.cqltypes.DateType) self.assertEqual(lookup_casstype('TimeType'), cassandra.cqltypes.TimeType) + self.assertEqual(lookup_casstype('TinyIntType'), cassandra.cqltypes.TinyIntType) + self.assertEqual(lookup_casstype('SmallIntType'), cassandra.cqltypes.SmallIntType) self.assertEqual(lookup_casstype('TimeUUIDType'), cassandra.cqltypes.TimeUUIDType) self.assertEqual(lookup_casstype('UUIDType'), cassandra.cqltypes.UUIDType) self.assertEqual(lookup_casstype('IntegerType'), cassandra.cqltypes.IntegerType) From dfa91b8bd5d6dcfe13ce3f4caf4eb721a7dd3d18 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 15 May 2015 14:46:06 -0500 Subject: [PATCH 0116/2431] Revert "Merge pull request #298 from datastax/PYTHON-108" Taking out timers in the interest of accelerating C* 2.2 features for a release. This reverts commit f879700dee6045b0c748a233253cf06bcce625e1, reversing changes made to fe4d2eaf2dd34809d823ab319a0cf82e4bd0bfa6. --- cassandra/cluster.py | 93 +++++++++----------- cassandra/connection.py | 106 ++--------------------- cassandra/io/asyncorereactor.py | 64 +++++++++----- cassandra/io/eventletreactor.py | 53 +++++------- cassandra/io/geventreactor.py | 56 +++++------- cassandra/io/libevreactor.py | 60 ++++++------- cassandra/io/libevwrapper.c | 133 ----------------------------- cassandra/io/twistedreactor.py | 62 ++++++-------- cassandra/query.py | 6 +- docs/object_mapper.rst | 2 +- tests/unit/test_connection.py | 3 + tests/unit/test_response_future.py | 22 ++--- 12 files changed, 201 insertions(+), 459 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 56e34e97dd..8f41948681 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -27,7 +27,6 @@ import sys import time from threading import Lock, RLock, Thread, Event -import warnings import six from six.moves import range @@ -72,7 +71,6 @@ BatchStatement, bind_params, QueryTrace, Statement, named_tuple_factory, dict_factory, FETCH_SIZE_UNSET) - def _is_eventlet_monkey_patched(): if 'eventlet.patcher' not in sys.modules: return False @@ -1267,7 +1265,8 @@ class Session(object): """ A default timeout, measured in seconds, for queries executed through :meth:`.execute()` or :meth:`.execute_async()`. This default may be - overridden with the `timeout` parameter for either of those methods. + overridden with the `timeout` parameter for either of those methods + or the `timeout` parameter for :meth:`.ResponseFuture.result()`. Setting this to :const:`None` will cause no timeouts to be set by default. @@ -1402,14 +1401,17 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): trace details, the :attr:`~.Statement.trace` attribute will be left as :const:`None`. """ + if timeout is _NOT_SET: + timeout = self.default_timeout + if trace and not isinstance(query, Statement): raise TypeError( "The query argument must be an instance of a subclass of " "cassandra.query.Statement when trace=True") - future = self.execute_async(query, parameters, trace, timeout) + future = self.execute_async(query, parameters, trace) try: - result = future.result() + result = future.result(timeout) finally: if trace: try: @@ -1419,7 +1421,7 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False): return result - def execute_async(self, query, parameters=None, trace=False, timeout=_NOT_SET): + def execute_async(self, query, parameters=None, trace=False): """ Execute the given query and return a :class:`~.ResponseFuture` object which callbacks may be attached to for asynchronous response @@ -1456,14 +1458,11 @@ def execute_async(self, query, parameters=None, trace=False, timeout=_NOT_SET): ... log.exception("Operation failed:") """ - if timeout is _NOT_SET: - timeout = self.default_timeout - - future = self._create_response_future(query, parameters, trace, timeout) + future = self._create_response_future(query, parameters, trace) future.send_request() return future - def _create_response_future(self, query, parameters, trace, timeout): + def _create_response_future(self, query, parameters, trace): """ Returns the ResponseFuture before calling send_request() on it """ prepared_statement = None @@ -1514,7 +1513,7 @@ def _create_response_future(self, query, parameters, trace, timeout): message.tracing = True return ResponseFuture( - self, message, query, timeout, metrics=self._metrics, + self, message, query, self.default_timeout, metrics=self._metrics, prepared_statement=prepared_statement) def prepare(self, query): @@ -1544,10 +1543,10 @@ def prepare(self, query): Preparing the same query more than once will likely affect performance. """ message = PrepareMessage(query=query) - future = ResponseFuture(self, message, query=None, timeout=self.default_timeout) + future = ResponseFuture(self, message, query=None) try: future.send_request() - query_id, column_metadata = future.result() + query_id, column_metadata = future.result(self.default_timeout) except Exception: log.exception("Error preparing query:") raise @@ -1572,7 +1571,7 @@ def prepare_on_all_hosts(self, query, excluded_host): futures = [] for host in self._pools.keys(): if host != excluded_host and host.is_up: - future = ResponseFuture(self, PrepareMessage(query=query), None, self.default_timeout) + future = ResponseFuture(self, PrepareMessage(query=query), None) # we don't care about errors preparing against specific hosts, # since we can always prepare them as needed when the prepared @@ -1593,7 +1592,7 @@ def prepare_on_all_hosts(self, query, excluded_host): for host, future in futures: try: - future.result() + future.result(self.default_timeout) except Exception: log.exception("Error preparing query for host %s:", host) @@ -2580,14 +2579,13 @@ class ResponseFuture(object): _start_time = None _metrics = None _paging_state = None - _timer = None - def __init__(self, session, message, query, timeout, metrics=None, prepared_statement=None): + def __init__(self, session, message, query, default_timeout=None, metrics=None, prepared_statement=None): self.session = session self.row_factory = session.row_factory self.message = message self.query = query - self.timeout = timeout + self.default_timeout = default_timeout self._metrics = metrics self.prepared_statement = prepared_statement self._callback_lock = Lock() @@ -2598,18 +2596,6 @@ def __init__(self, session, message, query, timeout, metrics=None, prepared_stat self._errors = {} self._callbacks = [] self._errbacks = [] - self._start_timer() - - def _start_timer(self): - if self.timeout is not None: - self._timer = self.session.cluster.connection_class.create_timer(self.timeout, self._on_timeout) - - def _cancel_timer(self): - if self._timer: - self._timer.cancel() - - def _on_timeout(self): - self._set_final_exception(OperationTimedOut(self._errors, self._current_host)) def _make_query_plan(self): # convert the list/generator/etc to an iterator so that subsequent @@ -2698,7 +2684,6 @@ def start_fetching_next_page(self): self._event.clear() self._final_result = _NOT_SET self._final_exception = None - self._start_timer() self.send_request() def _reprepare(self, prepare_message): @@ -2905,7 +2890,6 @@ def _execute_after_prepare(self, response): "statement on host %s: %s" % (self._current_host, response))) def _set_final_result(self, response): - self._cancel_timer() if self._metrics is not None: self._metrics.request_timer.addValue(time.time() - self._start_time) @@ -2920,7 +2904,6 @@ def _set_final_result(self, response): fn(response, *args, **kwargs) def _set_final_exception(self, response): - self._cancel_timer() if self._metrics is not None: self._metrics.request_timer.addValue(time.time() - self._start_time) @@ -2964,11 +2947,6 @@ def result(self, timeout=_NOT_SET): encountered. If the final result or error has not been set yet, this method will block until that time. - .. versionchanged:: 2.6.0 - - **`timeout` is deprecated. Use timeout in the Session execute functions instead. - The following description applies to deprecated behavior:** - You may set a timeout (in seconds) with the `timeout` parameter. By default, the :attr:`~.default_timeout` for the :class:`.Session` this was created through will be used for the timeout on this @@ -2982,6 +2960,11 @@ def result(self, timeout=_NOT_SET): This is a client-side timeout. For more information about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`. + **Important**: This timeout currently has no effect on callbacks registered + on a :class:`~.ResponseFuture` through :meth:`.ResponseFuture.add_callback` or + :meth:`.ResponseFuture.add_errback`; even if a query exceeds this default + timeout, neither the registered callback or errback will be called. + Example usage:: >>> future = session.execute_async("SELECT * FROM mycf") @@ -2995,24 +2978,27 @@ def result(self, timeout=_NOT_SET): ... log.exception("Operation failed:") """ - if timeout is not _NOT_SET: - msg = "ResponseFuture.result timeout argument is deprecated. Specify the request timeout via Session.execute[_async]." - warnings.warn(msg, DeprecationWarning) - log.warning(msg) - else: - timeout = None + if timeout is _NOT_SET: + timeout = self.default_timeout - self._event.wait(timeout) - # TODO: remove this conditional when deprecated timeout parameter is removed - if not self._event.is_set(): - self._on_timeout() if self._final_result is not _NOT_SET: if self._paging_state is None: return self._final_result else: - return PagedResult(self, self._final_result) - else: + return PagedResult(self, self._final_result, timeout) + elif self._final_exception: raise self._final_exception + else: + self._event.wait(timeout=timeout) + if self._final_result is not _NOT_SET: + if self._paging_state is None: + return self._final_result + else: + return PagedResult(self, self._final_result, timeout) + elif self._final_exception: + raise self._final_exception + else: + raise OperationTimedOut(errors=self._errors, last_host=self._current_host) def get_query_trace(self, max_wait=None): """ @@ -3162,9 +3148,10 @@ class will be returned. response_future = None - def __init__(self, response_future, initial_response): + def __init__(self, response_future, initial_response, timeout=_NOT_SET): self.response_future = response_future self.current_response = iter(initial_response) + self.timeout = timeout def __iter__(self): return self @@ -3177,7 +3164,7 @@ def next(self): raise self.response_future.start_fetching_next_page() - result = self.response_future.result() + result = self.response_future.result(self.timeout) if self.response_future.has_more_pages: self.current_response = result.current_response else: diff --git a/cassandra/connection.py b/cassandra/connection.py index 0d0c024b0d..c239313c45 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -16,12 +16,11 @@ from collections import defaultdict, deque import errno from functools import wraps, partial -from heapq import heappush, heappop import io import logging import os import sys -from threading import Thread, Event, RLock, Lock +from threading import Thread, Event, RLock import time if 'gevent.monkey' in sys.modules: @@ -39,8 +38,7 @@ QueryMessage, ResultMessage, decode_response, InvalidRequestException, SupportedMessage, AuthResponseMessage, AuthChallengeMessage, - AuthSuccessMessage, ProtocolException, - RegisterMessage) + AuthSuccessMessage, ProtocolException) from cassandra.util import OrderedDict @@ -194,7 +192,6 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.is_control_connection = is_control_connection self.user_type_map = user_type_map self._push_watchers = defaultdict(set) - self._callbacks = {} self._iobuf = io.BytesIO() if protocol_version >= 3: self._header_unpack = v3_header_unpack @@ -225,7 +222,6 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self._full_header_length = self._header_length + 4 self.lock = RLock() - self.connected_event = Event() @classmethod def initialize_reactor(self): @@ -260,10 +256,6 @@ def factory(cls, host, timeout, *args, **kwargs): else: return conn - @classmethod - def create_timer(cls, timeout, callback): - raise NotImplementedError() - def close(self): raise NotImplementedError() @@ -375,24 +367,11 @@ def wait_for_responses(self, *msgs, **kwargs): self.defunct(exc) raise - def register_watcher(self, event_type, callback, register_timeout=None): - """ - Register a callback for a given event type. - """ - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) + def register_watcher(self, event_type, callback): + raise NotImplementedError() - def register_watchers(self, type_callback_dict, register_timeout=None): - """ - Register multiple callback/event type pairs, expressed as a dict. - """ - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) + def register_watchers(self, type_callback_dict): + raise NotImplementedError() def control_conn_disposed(self): self.is_control_connection = False @@ -892,76 +871,3 @@ def stop(self): def _raise_if_stopped(self): if self._shutdown_event.is_set(): raise self.ShutdownException() - - -class Timer(object): - - canceled = False - - def __init__(self, timeout, callback): - self.end = time.time() + timeout - self.callback = callback - if timeout < 0: - self.callback() - - def cancel(self): - self.canceled = True - - def finish(self, time_now): - if self.canceled: - return True - - if time_now >= self.end: - self.callback() - return True - - return False - - -class TimerManager(object): - - def __init__(self): - self._queue = [] - self._new_timers = [] - - def add_timer(self, timer): - """ - called from client thread with a Timer object - """ - self._new_timers.append((timer.end, timer)) - - def service_timeouts(self): - """ - run callbacks on all expired timers - Called from the event thread - :return: next end time, or None - """ - queue = self._queue - new_timers = self._new_timers - while self._new_timers: - heappush(queue, new_timers.pop()) - now = time.time() - while queue: - try: - timer = queue[0][1] - if timer.finish(now): - heappop(queue) - else: - return timer.end - except Exception: - log.exception("Exception while servicing timeout callback: ") - - @property - def next_timeout(self): - try: - return self._queue[0][0] - except IndexError: - pass - - @property - def next_offset(self): - try: - next_end = self._queue[0][0] - return next_end - time.time() - except IndexError: - pass diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index b815d20aed..ef687c388c 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -19,12 +19,11 @@ import socket import sys from threading import Event, Lock, Thread -import time import weakref from six.moves import range -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN +from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN, errorcode try: from weakref import WeakSet except ImportError: @@ -37,9 +36,10 @@ except ImportError: ssl = None # NOQA +from cassandra import OperationTimedOut from cassandra.connection import (Connection, ConnectionShutdown, - ConnectionException, NONBLOCKING, - Timer, TimerManager) + ConnectionException, NONBLOCKING) +from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -55,17 +55,15 @@ def _cleanup(loop_weakref): class AsyncoreLoop(object): - def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() self._started = False self._shutdown = False + self._conns_lock = Lock() + self._conns = WeakSet() self._thread = None - - self._timers = TimerManager() - atexit.register(partial(_cleanup, weakref.ref(self))) def maybe_start(self): @@ -88,22 +86,24 @@ def maybe_start(self): def _run_loop(self): log.debug("Starting asyncore event loop") with self._loop_lock: - while not self._shutdown: + while True: try: - asyncore.loop(timeout=0.001, use_poll=True, count=100) - self._timers.service_timeouts() - if not asyncore.socket_map: - time.sleep(0.005) + asyncore.loop(timeout=0.001, use_poll=True, count=1000) except Exception: log.debug("Asyncore event loop stopped unexepectedly", exc_info=True) break + + if self._shutdown: + break + + with self._conns_lock: + if len(self._conns) == 0: + break + self._started = False log.debug("Asyncore event loop ended") - def add_timer(self, timer): - self._timers.add_timer(timer) - def _cleanup(self): self._shutdown = True if not self._thread: @@ -118,6 +118,14 @@ def _cleanup(self): log.debug("Event loop thread was joined") + def connection_created(self, connection): + with self._conns_lock: + self._conns.add(connection) + + def connection_destroyed(self, connection): + with self._conns_lock: + self._conns.discard(connection) + class AsyncoreConnection(Connection, asyncore.dispatcher): """ @@ -148,19 +156,18 @@ def handle_fork(cls): cls._loop._cleanup() cls._loop = None - @classmethod - def create_timer(cls, timeout, callback): - timer = Timer(timeout, callback) - cls._loop.add_timer(timer) - return timer - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) asyncore.dispatcher.__init__(self) + self.connected_event = Event() + + self._callbacks = {} self.deque = deque() self.deque_lock = Lock() + self._loop.connection_created(self) + sockerr = None addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) for (af, socktype, proto, canonname, sockaddr) in addresses: @@ -233,6 +240,8 @@ def close(self): asyncore.dispatcher.close(self) log.debug("Closed socket to %s", self.host) + self._loop.connection_destroyed(self) + if not self.is_defunct: self.error_all_callbacks( ConnectionShutdown("Connection to %s was closed" % self.host)) @@ -314,3 +323,14 @@ def writable(self): def readable(self): return self._readable or (self.is_control_connection and not (self.is_defunct or self.is_closed)) + + def register_watcher(self, event_type, callback, register_timeout=None): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), timeout=register_timeout) + + def register_watchers(self, type_callback_dict, register_timeout=None): + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), timeout=register_timeout) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index aceac55e10..670d0f1865 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -16,6 +16,7 @@ # Originally derived from MagnetoDB source: # https://github.com/stackforge/magnetodb/blob/2015.1.0b1/magnetodb/common/cassandra/io/eventletreactor.py +from collections import defaultdict from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL import eventlet from eventlet.green import select, socket @@ -24,11 +25,12 @@ import logging import os from threading import Event -import time from six.moves import xrange -from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager +from cassandra import OperationTimedOut +from cassandra.connection import Connection, ConnectionShutdown +from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -51,45 +53,19 @@ class EventletConnection(Connection): _write_watcher = None _socket = None - _timers = None - _timeout_watcher = None - _new_timer = None - @classmethod def initialize_reactor(cls): eventlet.monkey_patch() - if not cls._timers: - cls._timers = TimerManager() - cls._timeout_watcher = eventlet.spawn(cls.service_timeouts) - cls._new_timer = Event() - - @classmethod - def create_timer(cls, timeout, callback): - timer = Timer(timeout, callback) - cls._timers.add_timer(timer) - cls._new_timer.set() - return timer - - @classmethod - def service_timeouts(cls): - """ - cls._timeout_watcher runs in this loop forever. - It is usually waiting for the next timeout on the cls._new_timer Event. - When new timers are added, that event is set so that the watcher can - wake up and possibly set an earlier timeout. - """ - timer_manager = cls._timers - while True: - next_end = timer_manager.service_timeouts() - sleep_time = max(next_end - time.time(), 0) if next_end else 10000 - cls._new_timer.wait(sleep_time) - cls._new_timer.clear() def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) + self.connected_event = Event() self._write_queue = Queue() + self._callbacks = {} + self._push_watchers = defaultdict(set) + sockerr = None addresses = socket.getaddrinfo( self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM @@ -189,3 +165,16 @@ def push(self, data): chunk_size = self.out_buffer_size for i in xrange(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) + + def register_watcher(self, event_type, callback, register_timeout=None): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), + timeout=register_timeout) + + def register_watchers(self, type_callback_dict, register_timeout=None): + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), + timeout=register_timeout) diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 84285957bb..6e9af0da4d 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -13,20 +13,21 @@ # limitations under the License. import gevent from gevent import select, socket, ssl -import gevent.event +from gevent.event import Event from gevent.queue import Queue from collections import defaultdict from functools import partial import logging import os -import time -from six.moves import range +from six.moves import xrange from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager +from cassandra import OperationTimedOut +from cassandra.connection import Connection, ConnectionShutdown +from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -49,39 +50,15 @@ class GeventConnection(Connection): _write_watcher = None _socket = None - _timers = None - _timeout_watcher = None - _new_timer = None - - @classmethod - def initialize_reactor(cls): - if not cls._timers: - cls._timers = TimerManager() - cls._timeout_watcher = gevent.spawn(cls.service_timeouts) - cls._new_timer = gevent.event.Event() - - @classmethod - def create_timer(cls, timeout, callback): - timer = Timer(timeout, callback) - cls._timers.add_timer(timer) - cls._new_timer.set() - return timer - - @classmethod - def service_timeouts(cls): - timer_manager = cls._timers - timer_event = cls._new_timer - while True: - next_end = timer_manager.service_timeouts() - sleep_time = max(next_end - time.time(), 0) if next_end else 10000 - timer_event.wait(sleep_time) - timer_event.clear() - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) + self.connected_event = Event() self._write_queue = Queue() + self._callbacks = {} + self._push_watchers = defaultdict(set) + sockerr = None addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) for (af, socktype, proto, canonname, sockaddr) in addresses: @@ -182,5 +159,18 @@ def handle_read(self): def push(self, data): chunk_size = self.out_buffer_size - for i in range(0, len(data), chunk_size): + for i in xrange(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) + + def register_watcher(self, event_type, callback, register_timeout=None): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), + timeout=register_timeout) + + def register_watchers(self, type_callback_dict, register_timeout=None): + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), + timeout=register_timeout) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index e6abb76b41..93b4c97854 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -20,10 +20,11 @@ from threading import Event, Lock, Thread import weakref -from six.moves import range +from six.moves import xrange -from cassandra.connection import (Connection, ConnectionShutdown, - NONBLOCKING, Timer, TimerManager) +from cassandra import OperationTimedOut +from cassandra.connection import Connection, ConnectionShutdown, NONBLOCKING +from cassandra.protocol import RegisterMessage try: import cassandra.io.libevwrapper as libev except ImportError: @@ -39,7 +40,7 @@ try: import ssl except ImportError: - ssl = None # NOQA + ssl = None # NOQA log = logging.getLogger(__name__) @@ -49,6 +50,7 @@ def _cleanup(loop_weakref): loop = loop_weakref() except ReferenceError: return + loop._cleanup() @@ -83,11 +85,11 @@ def __init__(self): self._loop.unref() self._preparer.start() - self._timers = TimerManager() - self._loop_timer = libev.Timer(self._loop, self._on_loop_timer) - atexit.register(partial(_cleanup, weakref.ref(self))) + def notify(self): + self._notifier.send() + def maybe_start(self): should_start = False with self._lock: @@ -131,7 +133,6 @@ def _cleanup(self): conn._read_watcher.stop() del conn._read_watcher - self.notify() # wake the timer watcher log.debug("Waiting for event loop thread to join...") self._thread.join(timeout=1.0) if self._thread.is_alive(): @@ -142,24 +143,6 @@ def _cleanup(self): log.debug("Event loop thread was joined") self._loop = None - def add_timer(self, timer): - self._timers.add_timer(timer) - self._notifier.send() # wake up in case this timer is earlier - - def _update_timer(self): - if not self._shutdown: - self._timers.service_timeouts() - offset = self._timers.next_offset or 100000 # none pending; will be updated again when something new happens - self._loop_timer.start(offset) - else: - self._loop_timer.stop() - - def _on_loop_timer(self): - self._timers.service_timeouts() - - def notify(self): - self._notifier.send() - def connection_created(self, conn): with self._conn_set_lock: new_live_conns = self._live_conns.copy() @@ -222,9 +205,6 @@ def _loop_will_run(self, prepare): changed = True - # TODO: update to do connection management, timer updates through dedicaterd async 'notifier' callbacks - self._update_timer() - if changed: self._notifier.send() @@ -256,15 +236,12 @@ def handle_fork(cls): cls._libevloop._cleanup() cls._libevloop = None - @classmethod - def create_timer(cls, timeout, callback): - timer = Timer(timeout, callback) - cls._libevloop.add_timer(timer) - return timer - def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) + self.connected_event = Event() + + self._callbacks = {} self.deque = deque() self._deque_lock = Lock() @@ -384,7 +361,7 @@ def push(self, data): sabs = self.out_buffer_size if len(data) > sabs: chunks = [] - for i in range(0, len(data), sabs): + for i in xrange(0, len(data), sabs): chunks.append(data[i:i + sabs]) else: chunks = [data] @@ -392,3 +369,14 @@ def push(self, data): with self._deque_lock: self.deque.extend(chunks) self._libevloop.notify() + + def register_watcher(self, event_type, callback, register_timeout=None): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), timeout=register_timeout) + + def register_watchers(self, type_callback_dict, register_timeout=None): + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), timeout=register_timeout) diff --git a/cassandra/io/libevwrapper.c b/cassandra/io/libevwrapper.c index 99e1df30f7..cbac83b277 100644 --- a/cassandra/io/libevwrapper.c +++ b/cassandra/io/libevwrapper.c @@ -451,131 +451,6 @@ static PyTypeObject libevwrapper_PrepareType = { (initproc)Prepare_init, /* tp_init */ }; -typedef struct libevwrapper_Timer { - PyObject_HEAD - struct ev_timer timer; - struct libevwrapper_Loop *loop; - PyObject *callback; -} libevwrapper_Timer; - -static void -Timer_dealloc(libevwrapper_Timer *self) { - Py_XDECREF(self->loop); - Py_XDECREF(self->callback); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static void timer_callback(struct ev_loop *loop, ev_timer *watcher, int revents) { - libevwrapper_Timer *self = watcher->data; - - PyObject *result = NULL; - PyGILState_STATE gstate; - - gstate = PyGILState_Ensure(); - result = PyObject_CallFunction(self->callback, NULL); - if (!result) { - PyErr_WriteUnraisable(self->callback); - } - Py_XDECREF(result); - - PyGILState_Release(gstate); -} - -static int -Timer_init(libevwrapper_Timer *self, PyObject *args, PyObject *kwds) { - PyObject *callback; - PyObject *loop; - - if (!PyArg_ParseTuple(args, "OO", &loop, &callback)) { - return -1; - } - - if (loop) { - Py_INCREF(loop); - self->loop = (libevwrapper_Loop *)loop; - } else { - return -1; - } - - if (callback) { - if (!PyCallable_Check(callback)) { - PyErr_SetString(PyExc_TypeError, "callback parameter must be callable"); - Py_XDECREF(loop); - return -1; - } - Py_INCREF(callback); - self->callback = callback; - } - ev_init(&self->timer, timer_callback); - self->timer.data = self; - return 0; -} - -static PyObject * -Timer_start(libevwrapper_Timer *self, PyObject *args) { - double timeout; - if (!PyArg_ParseTuple(args, "d", &timeout)) { - return NULL; - } - /* some tiny non-zero number to avoid zero, and - make it run immediately for negative timeouts */ - self->timer.repeat = fmax(timeout, 0.000000001); - ev_timer_again(self->loop->loop, &self->timer); - Py_RETURN_NONE; -} - -static PyObject * -Timer_stop(libevwrapper_Timer *self, PyObject *args) { - ev_timer_stop(self->loop->loop, &self->timer); - Py_RETURN_NONE; -} - -static PyMethodDef Timer_methods[] = { - {"start", (PyCFunction)Timer_start, METH_VARARGS, "Start the Timer watcher"}, - {"stop", (PyCFunction)Timer_stop, METH_NOARGS, "Stop the Timer watcher"}, - {NULL} /* Sentinal */ -}; - -static PyTypeObject libevwrapper_TimerType = { - PyVarObject_HEAD_INIT(NULL, 0) - "cassandra.io.libevwrapper.Timer", /*tp_name*/ - sizeof(libevwrapper_Timer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Timer_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "Timer objects", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Timer_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Timer_init, /* tp_init */ -}; - - static PyMethodDef module_methods[] = { {NULL} /* Sentinal */ }; @@ -625,10 +500,6 @@ initlibevwrapper(void) if (PyType_Ready(&libevwrapper_AsyncType) < 0) INITERROR; - libevwrapper_TimerType.tp_new = PyType_GenericNew; - if (PyType_Ready(&libevwrapper_TimerType) < 0) - INITERROR; - # if PY_MAJOR_VERSION >= 3 module = PyModule_Create(&moduledef); # else @@ -661,10 +532,6 @@ initlibevwrapper(void) if (PyModule_AddObject(module, "Async", (PyObject *)&libevwrapper_AsyncType) == -1) INITERROR; - Py_INCREF(&libevwrapper_TimerType); - if (PyModule_AddObject(module, "Timer", (PyObject *)&libevwrapper_TimerType) == -1) - INITERROR; - if (!PyEval_ThreadsInitialized()) { PyEval_InitThreads(); } diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index 0f5c841c75..ff81e5613f 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -15,15 +15,16 @@ Module that implements an event loop based on twisted ( https://twistedmatrix.com ). """ -import atexit +from twisted.internet import reactor, protocol +from threading import Event, Thread, Lock from functools import partial import logging -from threading import Event, Thread, Lock -import time -from twisted.internet import reactor, protocol import weakref +import atexit -from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager +from cassandra import OperationTimedOut +from cassandra.connection import Connection, ConnectionShutdown +from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -108,12 +109,9 @@ class TwistedLoop(object): _lock = None _thread = None - _timeout_task = None - _timeout = None def __init__(self): self._lock = Lock() - self._timers = TimerManager() def maybe_start(self): with self._lock: @@ -135,27 +133,6 @@ def _cleanup(self): "Cluster.shutdown() to avoid this.") log.debug("Event loop thread was joined") - def add_timer(self, timer): - self._timers.add_timer(timer) - # callFromThread to schedule from the loop thread, where - # the timeout task can safely be modified - reactor.callFromThread(self._schedule_timeout, timer.end) - - def _schedule_timeout(self, next_timeout): - if next_timeout: - delay = max(next_timeout - time.time(), 0) - if self._timeout_task and self._timeout_task.active(): - if next_timeout < self._timeout: - self._timeout_task.reset(delay) - self._timeout = next_timeout - else: - self._timeout_task = reactor.callLater(delay, self._on_loop_timer) - self._timeout = next_timeout - - def _on_loop_timer(self): - self._timers.service_timeouts() - self._schedule_timeout(self._timers.next_timeout) - class TwistedConnection(Connection): """ @@ -171,12 +148,6 @@ def initialize_reactor(cls): if not cls._loop: cls._loop = TwistedLoop() - @classmethod - def create_timer(cls, timeout, callback): - timer = Timer(timeout, callback) - cls._loop.add_timer(timer) - return timer - def __init__(self, *args, **kwargs): """ Initialization method. @@ -188,9 +159,11 @@ def __init__(self, *args, **kwargs): """ Connection.__init__(self, *args, **kwargs) + self.connected_event = Event() self.is_closed = True self.connector = None + self._callbacks = {} reactor.callFromThread(self.add_connection) self._loop.maybe_start() @@ -247,3 +220,22 @@ def push(self, data): the event loop when it gets the chance. """ reactor.callFromThread(self.connector.transport.write, data) + + def register_watcher(self, event_type, callback, register_timeout=None): + """ + Register a callback for a given event type. + """ + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), + timeout=register_timeout) + + def register_watchers(self, type_callback_dict, register_timeout=None): + """ + Register multiple callback/event type pairs, expressed as a dict. + """ + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), + timeout=register_timeout) diff --git a/cassandra/query.py b/cassandra/query.py index 30bf450117..1232514bfa 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -844,14 +844,14 @@ def populate(self, max_wait=2.0): break def _execute(self, query, parameters, time_spent, max_wait): - timeout = (max_wait - time_spent) if max_wait is not None else None - future = self._session._create_response_future(query, parameters, trace=False, timeout=timeout) # in case the user switched the row factory, set it to namedtuple for this query + future = self._session._create_response_future(query, parameters, trace=False) future.row_factory = named_tuple_factory future.send_request() + timeout = (max_wait - time_spent) if max_wait is not None else None try: - return future.result() + return future.result(timeout=timeout) except OperationTimedOut: raise TraceUnavailable("Trace information was not available within %f seconds" % (max_wait,)) diff --git a/docs/object_mapper.rst b/docs/object_mapper.rst index 4e38994064..26d78a0964 100644 --- a/docs/object_mapper.rst +++ b/docs/object_mapper.rst @@ -48,7 +48,7 @@ Getting Started from cassandra.cqlengine import columns from cassandra.cqlengine import connection from datetime import datetime - from cassandra.cqlengine.management import sync_table + from cassandra.cqlengine.management import create_keyspace, sync_table from cassandra.cqlengine.models import Model #first, define a model diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index a779e1338c..7fc4ed4e5a 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -261,7 +261,10 @@ def test_not_implemented(self): Ensure the following methods throw NIE's. If not, come back and test them. """ c = self.make_connection() + self.assertRaises(NotImplementedError, c.close) + self.assertRaises(NotImplementedError, c.register_watcher, None, None) + self.assertRaises(NotImplementedError, c.register_watchers, None) def test_set_keyspace_blocking(self): c = self.make_connection() diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index 986945ba4a..027fe73214 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -47,7 +47,7 @@ def make_session(self): def make_response_future(self, session): query = SimpleStatement("SELECT * FROM foo") message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - return ResponseFuture(session, message, query, 1) + return ResponseFuture(session, message, query) def make_mock_response(self, results): return Mock(spec=ResultMessage, kind=RESULT_KIND_ROWS, results=results, paging_state=None) @@ -122,7 +122,7 @@ def test_read_timeout_error_message(self): query.retry_policy.on_read_timeout.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() result = Mock(spec=ReadTimeoutErrorMessage, info={}) @@ -137,7 +137,7 @@ def test_write_timeout_error_message(self): query.retry_policy.on_write_timeout.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() result = Mock(spec=WriteTimeoutErrorMessage, info={}) @@ -151,7 +151,7 @@ def test_unavailable_error_message(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() result = Mock(spec=UnavailableErrorMessage, info={}) @@ -165,7 +165,7 @@ def test_retry_policy_says_ignore(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.IGNORE, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() result = Mock(spec=UnavailableErrorMessage, info={}) @@ -184,7 +184,7 @@ def test_retry_policy_says_retry(self): connection = Mock(spec=Connection) pool.borrow_connection.return_value = (connection, 1) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() rf.session._pools.get.assert_called_once_with('ip1') @@ -279,7 +279,7 @@ def test_all_pools_shutdown(self): session._load_balancer.make_query_plan.return_value = ['ip1', 'ip2'] session._pools.get.return_value.is_shutdown = True - rf = ResponseFuture(session, Mock(), Mock(), 1) + rf = ResponseFuture(session, Mock(), Mock()) rf.send_request() self.assertRaises(NoHostAvailable, rf.result) @@ -354,7 +354,7 @@ def test_errback(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() rf.add_errback(self.assertIsInstance, Exception) @@ -401,7 +401,7 @@ def test_multiple_errbacks(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() callback = Mock() @@ -431,7 +431,7 @@ def test_add_callbacks(self): message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) # test errback - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() rf.add_callbacks( @@ -443,7 +443,7 @@ def test_add_callbacks(self): self.assertRaises(Exception, rf.result) # test callback - rf = ResponseFuture(session, message, query, 1) + rf = ResponseFuture(session, message, query) rf.send_request() callback = Mock() From 0dcfa5e24fc3b6b0b95f71e4c13adaed01e50f67 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 15 May 2015 15:34:25 -0500 Subject: [PATCH 0117/2431] Change Cassandra 3.0 references to Cassandra 2.2 --- cassandra/cluster.py | 4 ++-- cassandra/metadata.py | 12 ++++++------ cassandra/query.py | 8 ++++---- tests/integration/__init__.py | 2 +- tests/integration/standard/test_query.py | 16 ++++++++-------- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 9dee46150b..fe7e3372c4 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2212,7 +2212,7 @@ def _handle_results(success, result): else: raise types_result - # functions were introduced in Cassandra 3.0 + # functions were introduced in Cassandra 2.2 if functions_success: functions_result = dict_factory(*functions_result.results) if functions_result.results else {} else: @@ -2222,7 +2222,7 @@ def _handle_results(success, result): else: raise functions_result - # aggregates were introduced in Cassandra 3.0 + # aggregates were introduced in Cassandra 2.2 if aggregates_success: aggregates_result = dict_factory(*aggregates_result.results) if aggregates_result.results else {} else: diff --git a/cassandra/metadata.py b/cassandra/metadata.py index fe05304fd3..0d872d5b93 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -800,14 +800,14 @@ class KeyspaceMetadata(object): """ A map from user-defined function signatures to instances of :class:`~cassandra.metadata.Function`. - .. versionadded:: 3.0.0 + .. versionadded:: 2.6.0 """ aggregates = None """ A map from user-defined aggregate signatures to instances of :class:`~cassandra.metadata.Aggregate`. - .. versionadded:: 3.0.0 + .. versionadded:: 2.6.0 """ def __init__(self, name, durable_writes, strategy_class, strategy_options): self.name = name @@ -935,9 +935,9 @@ class Aggregate(object): """ A user defined aggregate function, as created by ``CREATE AGGREGATE`` statements. - Aggregate functions were introduced in Cassandra 3.0 + Aggregate functions were introduced in Cassandra 2.2 - .. versionadded:: 3.0.0 + .. versionadded:: 2.6.0 """ keyspace = None @@ -1024,9 +1024,9 @@ class Function(object): """ A user defined function, as created by ``CREATE FUNCTION`` statements. - User-defined functions were introduced in Cassandra 3.0 + User-defined functions were introduced in Cassandra 2.2 - .. versionadded:: 3.0.0 + .. versionadded:: 2.6.0 """ keyspace = None diff --git a/cassandra/query.py b/cassandra/query.py index f5c46c53e2..058ba14494 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -209,7 +209,7 @@ class Statement(object): These are only allowed when using protocol version 4 or higher. - .. versionadded:: 3.0.0 + .. versionadded:: 2.6.0 """ _serial_consistency_level = None @@ -666,7 +666,7 @@ def __init__(self, batch_type=BatchType.LOGGED, retry_policy=None, .. versionchanged:: 2.1.0 Added `serial_consistency_level` as a parameter - .. versionchanged:: 3.0.0 + .. versionchanged:: 2.6.0 Added `custom_payload` as a parameter """ self.batch_type = batch_type @@ -801,7 +801,7 @@ class QueryTrace(object): """ The IP address of the client that issued this request - This is only available when using Cassandra 3.0+ + This is only available when using Cassandra 2.2+ """ coordinator = None @@ -872,7 +872,7 @@ def populate(self, max_wait=2.0): self.started_at = session_row.started_at self.coordinator = session_row.coordinator self.parameters = session_row.parameters - # since C* 3.0 + # since C* 2.2 self.client = getattr(session_row, 'client', None) log.debug("Attempting to fetch trace events for trace ID: %s", self.trace_id) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 18a99c04de..7f0cd3bd9d 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -119,7 +119,7 @@ def _tuple_version(version_string): log.info('Using Cassandra version: %s', CASSANDRA_VERSION) CCM_KWARGS['version'] = CASSANDRA_VERSION -if CASSANDRA_VERSION >= '3.0': +if CASSANDRA_VERSION >= '2.2': default_protocol_version = 4 elif CASSANDRA_VERSION >= '2.1': default_protocol_version = 3 diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 755e897a41..b5d07f4487 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -108,21 +108,21 @@ def test_client_ip_in_trace(self): Test to validate that client trace contains client ip information. creates a simple query and ensures that the client trace information is present. This will - only be the case if the c* version is 3.0 or greater + only be the case if the c* version is 2.2 or greater - @since 3.0 + @since 2.6.0 @jira_ticket PYTHON-235 - @expected_result client address should be present in C* >= 3, otherwise should be none. + @expected_result client address should be present in C* >= 2.2, otherwise should be none. @test_category tracing + """ - #The current version on the trunk doesn't have the version set to 3.0 yet. + #The current version on the trunk doesn't have the version set to 2.2 yet. #For now we will use the protocol version. Once they update the version on C* trunk #we can use the C*. See below #self._cass_version, self._cql_version = get_server_versions() - #if self._cass_version < (3, 0): - # raise unittest.SkipTest("Client IP was not present in trace until C* 3.0") + #if self._cass_version < (2, 2): + # raise unittest.SkipTest("Client IP was not present in trace until C* 2.2") if PROTOCOL_VERSION < 4: raise unittest.SkipTest( "Protocol 4+ is required for client ip tracing, currently testing against %r" @@ -139,8 +139,8 @@ def test_client_ip_in_trace(self): # Fetch the client_ip from the trace. trace = response_future.get_query_trace(2.0) client_ip = trace.client - # Ensure that ip is set for c* >3 - self.assertIsNotNone(client_ip,"Client IP was not set in trace with C* > 3.0") + # Ensure that ip is set + self.assertIsNotNone(client_ip,"Client IP was not set in trace with C* >= 2.2") self.assertEqual(client_ip,current_host,"Client IP from trace did not match the expected value") cluster.shutdown() From 0646d15fa1c5cff5247750edcf2935ab62242dec Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 15 May 2015 16:25:52 -0500 Subject: [PATCH 0118/2431] Remove is_deterministic from function meta It was removed in the final version of the server changes. Also skipping legacy table test for 2.2, which does not have cassandra-cli --- cassandra/metadata.py | 17 ++++------------ tests/integration/__init__.py | 3 ++- tests/integration/standard/test_metadata.py | 22 +++++++-------------- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 0d872d5b93..ece39cec16 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -250,7 +250,7 @@ def _build_function(self, keyspace, function_row): return Function(function_row['keyspace_name'], function_row['function_name'], function_row['signature'], function_row['argument_names'], return_type, function_row['language'], function_row['body'], - function_row['is_deterministic'], function_row['called_on_null_input']) + function_row['called_on_null_input']) def _build_aggregate(self, keyspace, aggregate_row): state_type = types.lookup_casstype(aggregate_row['state_type']) @@ -977,8 +977,7 @@ class Aggregate(object): state_type = None """ - Flag indicating whether this function is deterministic - (required for functional indexes) + Type of the aggregate state """ def __init__(self, keyspace, name, type_signature, state_func, @@ -1064,12 +1063,6 @@ class Function(object): Function body string """ - is_deterministic = None - """ - Flag indicating whether this function is deterministic - (required for functional indexes) - """ - called_on_null_input = None """ Flag indicating whether this function should be called for rows with null values @@ -1077,7 +1070,7 @@ class Function(object): """ def __init__(self, keyspace, name, type_signature, argument_names, - return_type, language, body, is_deterministic, called_on_null_input): + return_type, language, body, called_on_null_input): self.keyspace = keyspace self.name = name self.type_signature = type_signature @@ -1085,7 +1078,6 @@ def __init__(self, keyspace, name, type_signature, argument_names, self.return_type = return_type self.language = language self.body = body - self.is_deterministic = is_deterministic self.called_on_null_input = called_on_null_input def as_cql_query(self, formatted=False): @@ -1099,13 +1091,12 @@ def as_cql_query(self, formatted=False): name = protect_name(self.name) arg_list = ', '.join(["%s %s" % (protect_name(n), t) for n, t in zip(self.argument_names, self.type_signature)]) - determ = '' if self.is_deterministic else 'NON DETERMINISTIC ' typ = self.return_type.cql_parameterized_type() lang = self.language body = protect_value(self.body) on_null = "CALLED" if self.called_on_null_input else "RETURNS NULL" - return "CREATE %(determ)sFUNCTION %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ + return "CREATE FUNCTION %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ "%(on_null)s ON NULL INPUT%(sep)s" \ "RETURNS %(typ)s%(sep)s" \ "LANGUAGE %(lang)s%(sep)s" \ diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 7f0cd3bd9d..ddce8de86f 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -197,7 +197,8 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True): except Exception: log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS) cluster = CCMCluster(path, cluster_name, **CCM_KWARGS) - cluster.set_configuration_options({'start_native_transport': True}) + cluster.set_configuration_options({'start_native_transport': True, + 'enable_user_defined_functions': True}) common.switch_cluster(path, cluster_name) cluster.populate(nodes, ipformat=ipformat) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 003c53a955..3f3ef71092 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -568,9 +568,14 @@ def test_token_map(self): def test_legacy_tables(self): - if get_server_versions()[0] < (2, 1, 0): + cass_ver = get_server_versions()[0] + print cass_ver + if cass_ver < (2, 1, 0): raise unittest.SkipTest('Test schema output assumes 2.1.0+ options') + if cass_ver >= (2, 2, 0): + raise unittest.SkipTest('Cannot test cli script on Cassandra 2.2.0+') + if sys.version_info[0:2] != (2, 7): raise unittest.SkipTest('This test compares static strings generated from dict items, which may change orders. Test with 2.7.') @@ -1075,7 +1080,7 @@ def __init__(self, test_case, **kwargs): class FunctionMetadata(FunctionTest): - def make_function_kwargs(self, deterministic=True, called_on_null=True): + def make_function_kwargs(self, called_on_null=True): return {'keyspace': self.keyspace_name, 'name': self.function_name, 'type_signature': ['double', 'int'], @@ -1083,7 +1088,6 @@ def make_function_kwargs(self, deterministic=True, called_on_null=True): 'return_type': DoubleType, 'language': 'java', 'body': 'return new Double(0.0);', - 'is_deterministic': deterministic, 'called_on_null_input': called_on_null} def test_functions_after_udt(self): @@ -1133,18 +1137,6 @@ def test_functions_follow_keyspace_alter(self): finally: self.session.execute('ALTER KEYSPACE %s WITH durable_writes = true' % self.keyspace_name) - def test_function_cql_determinism(self): - kwargs = self.make_function_kwargs() - kwargs['is_deterministic'] = True - with self.VerifiedFunction(self, **kwargs) as vf: - fn_meta = self.keyspace_function_meta[vf.signature] - self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE FUNCTION.*") - - kwargs['is_deterministic'] = False - with self.VerifiedFunction(self, **kwargs) as vf: - fn_meta = self.keyspace_function_meta[vf.signature] - self.assertRegexpMatches(fn_meta.as_cql_query(), "CREATE NON DETERMINISTIC FUNCTION.*") - def test_function_cql_called_on_null(self): kwargs = self.make_function_kwargs() kwargs['called_on_null_input'] = True From ebfa77c19d98c5a5a90a31db493794aee295af45 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 15 May 2015 16:47:20 -0500 Subject: [PATCH 0119/2431] refresh_type_metadata -> refresh_user_type_metadata code review input --- cassandra/cluster.py | 2 +- docs/api/cassandra/cluster.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index c48a1ebffd..d9be588b6d 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1202,7 +1202,7 @@ def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None if not self.control_connection.refresh_schema(keyspace, table, schema_agreement_wait=max_schema_agreement_wait): raise Exception("Table metadata was not refreshed. See log for details.") - def refresh_type_metadata(self, keyspace, user_type, max_schema_agreement_wait=None): + def refresh_user_type_metadata(self, keyspace, user_type, max_schema_agreement_wait=None): """ Synchronously refresh user defined type metadata. diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 886d07160e..2174a8169f 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -71,7 +71,7 @@ .. automethod:: refresh_table_metadata - .. automethod:: refresh_type_metadata + .. automethod:: refresh_user_type_metadata .. automethod:: refresh_schema From f05921a583a84c0c749f01f5a596c8240315cab4 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 15 May 2015 16:49:37 -0500 Subject: [PATCH 0120/2431] cqle: Make CQL UPPER in management.drop_table code review input --- cassandra/cqlengine/management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/cqlengine/management.py b/cassandra/cqlengine/management.py index d84406f959..b5d2cb4620 100644 --- a/cassandra/cqlengine/management.py +++ b/cassandra/cqlengine/management.py @@ -523,7 +523,7 @@ def drop_table(model): try: meta.keyspaces[ks_name].tables[raw_cf_name] - execute('drop table {};'.format(model.column_family_name())) + execute('DROP TABLE {};'.format(model.column_family_name())) except KeyError: pass From ffaaddcf133fb770528611670bca4fd13350feb9 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 15 May 2015 16:16:07 -0700 Subject: [PATCH 0121/2431] Unbreak tests by disabling UDFs in cassandra.yaml --- tests/integration/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index ddce8de86f..5c09af3bc9 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -197,8 +197,9 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True): except Exception: log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS) cluster = CCMCluster(path, cluster_name, **CCM_KWARGS) - cluster.set_configuration_options({'start_native_transport': True, - 'enable_user_defined_functions': True}) + cluster.set_configuration_options({'start_native_transport': True}) + if CASSANDRA_VERSION >= '2.2': + cluster.set_configuration_options({'enable_user_defined_functions': True}) common.switch_cluster(path, cluster_name) cluster.populate(nodes, ipformat=ipformat) From 830958952b40e2ae972e5fecde3c0358d905fc00 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 15 May 2015 16:51:03 -0700 Subject: [PATCH 0122/2431] temporarily synchronously refresh the schema metadata, for IndexMapTests --- tests/integration/standard/test_metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 3f3ef71092..fea92f2fb4 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -981,6 +981,10 @@ def test_index_updates(self): # both indexes updated when index dropped self.session.execute("DROP INDEX a_idx") + + # temporarily synchronously refresh the schema metadata, until CASSANDRA-9391 is merged in + self.cluster.refresh_schema(self.keyspace_name, self.table_name) + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] table_meta = ks_meta.tables[self.table_name] self.assertNotIn('a_idx', ks_meta.indexes) From dcb41c3c9a18eb7aa6eafd5f59f9083fdcf1e9b2 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 15 May 2015 17:18:33 -0700 Subject: [PATCH 0123/2431] Unbreak UDF metadata tests, run only if protocol v 4+ --- tests/integration/standard/test_metadata.py | 87 ++++++++++++--------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index fea92f2fb4..612593f597 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -1022,27 +1022,35 @@ class FunctionTest(unittest.TestCase): """ Base functionality for Function and Aggregate metadata test classes """ + + def setUp(self): + """ + Tests are skipped if run with native protocol version < 4 + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Function metadata requires native protocol version 4+") + @property def function_name(self): return self._testMethodName.lower() @classmethod def setup_class(cls): - if PROTOCOL_VERSION < 4: - raise unittest.SkipTest("Function metadata requires native protocol version 4+") - - cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) - cls.keyspace_name = cls.__name__.lower() - cls.session = cls.cluster.connect() - cls.session.execute("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}" % cls.keyspace_name) - cls.session.set_keyspace(cls.keyspace_name) - cls.keyspace_function_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].functions - cls.keyspace_aggregate_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].aggregates + if PROTOCOL_VERSION >= 4: + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.keyspace_name = cls.__name__.lower() + cls.session = cls.cluster.connect() + cls.session.execute("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}" % cls.keyspace_name) + cls.session.set_keyspace(cls.keyspace_name) + cls.keyspace_function_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].functions + cls.keyspace_aggregate_meta = cls.cluster.metadata.keyspaces[cls.keyspace_name].aggregates @classmethod def teardown_class(cls): - cls.session.execute("DROP KEYSPACE IF EXISTS %s" % cls.keyspace_name) - cls.cluster.shutdown() + if PROTOCOL_VERSION >= 4: + cls.session.execute("DROP KEYSPACE IF EXISTS %s" % cls.keyspace_name) + cls.cluster.shutdown() class Verified(object): @@ -1158,33 +1166,34 @@ class AggregateMetadata(FunctionTest): @classmethod def setup_class(cls): - super(AggregateMetadata, cls).setup_class() - - cls.session.execute("""CREATE OR REPLACE FUNCTION sum_int(s int, i int) - RETURNS NULL ON NULL INPUT - RETURNS int - LANGUAGE javascript AS 's + i';""") - cls.session.execute("""CREATE OR REPLACE FUNCTION sum_int_two(s int, i int, j int) - RETURNS NULL ON NULL INPUT - RETURNS int - LANGUAGE javascript AS 's + i + j';""") - cls.session.execute("""CREATE OR REPLACE FUNCTION "List_As_String"(l list) - RETURNS NULL ON NULL INPUT - RETURNS int - LANGUAGE javascript AS ''''' + l';""") - cls.session.execute("""CREATE OR REPLACE FUNCTION extend_list(s list, i int) - CALLED ON NULL INPUT - RETURNS list - LANGUAGE java AS 'if (i != null) s.add(i.toString()); return s;';""") - cls.session.execute("""CREATE OR REPLACE FUNCTION update_map(s map, i int) - RETURNS NULL ON NULL INPUT - RETURNS map - LANGUAGE java AS 's.put(new Integer(i), new Integer(i)); return s;';""") - cls.session.execute("""CREATE TABLE IF NOT EXISTS t - (k int PRIMARY KEY, v int)""") - for x in range(4): - cls.session.execute("INSERT INTO t (k,v) VALUES (%s, %s)", (x, x)) - cls.session.execute("INSERT INTO t (k) VALUES (%s)", (4,)) + if PROTOCOL_VERSION >= 4: + super(AggregateMetadata, cls).setup_class() + + cls.session.execute("""CREATE OR REPLACE FUNCTION sum_int(s int, i int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 's + i';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION sum_int_two(s int, i int, j int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 's + i + j';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION "List_As_String"(l list) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS ''''' + l';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION extend_list(s list, i int) + CALLED ON NULL INPUT + RETURNS list + LANGUAGE java AS 'if (i != null) s.add(i.toString()); return s;';""") + cls.session.execute("""CREATE OR REPLACE FUNCTION update_map(s map, i int) + RETURNS NULL ON NULL INPUT + RETURNS map + LANGUAGE java AS 's.put(new Integer(i), new Integer(i)); return s;';""") + cls.session.execute("""CREATE TABLE IF NOT EXISTS t + (k int PRIMARY KEY, v int)""") + for x in range(4): + cls.session.execute("INSERT INTO t (k,v) VALUES (%s, %s)", (x, x)) + cls.session.execute("INSERT INTO t (k) VALUES (%s)", (4,)) def make_aggregate_kwargs(self, state_func, state_type, final_func=None, init_cond=None): return {'keyspace': self.keyspace_name, From b004dc6ff0669d12806cca53b4db38cf27c2ffc2 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 15 May 2015 17:43:02 -0700 Subject: [PATCH 0124/2431] remove extra printline in TestCodeCoverage --- tests/integration/standard/test_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 612593f597..73ffe134ce 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -569,7 +569,6 @@ def test_token_map(self): def test_legacy_tables(self): cass_ver = get_server_versions()[0] - print cass_ver if cass_ver < (2, 1, 0): raise unittest.SkipTest('Test schema output assumes 2.1.0+ options') From 9b940d25933a5bb4fd633ad388a77bf28b3e1044 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 18 May 2015 08:22:58 -0500 Subject: [PATCH 0125/2431] Correct expected values for Small|TinyInt unit tests --- tests/unit/test_marshalling.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_marshalling.py b/tests/unit/test_marshalling.py index eeefb0f174..907cd765b6 100644 --- a/tests/unit/test_marshalling.py +++ b/tests/unit/test_marshalling.py @@ -83,9 +83,9 @@ (b'\x7f\xff\xff\xff', 'SimpleDateType', Date('1969-12-31')), (b'\x00\x00\x00\x00\x00\x00\x00\x01', 'TimeType', Time(1)), (b'\x7f', 'TinyIntType', 127), - (b'\xff\xff\xff\x80', 'TinyIntType', -128), - (b'\xff\xff\x80\x00', 'SmallIntType', 32767), - (b'\xff\xff\x80\x00', 'SmallIntType', -32768) + (b'\x80', 'TinyIntType', -128), + (b'\x7f\xff', 'SmallIntType', 32767), + (b'\x80\x00', 'SmallIntType', -32768) ) ordered_map_value = OrderedMapSerializedKey(UTF8Type, 2) From 65d14ddb7b9c119feae59c760a43fa5b459c48ea Mon Sep 17 00:00:00 2001 From: GregBestland Date: Tue, 12 May 2015 00:29:49 -0500 Subject: [PATCH 0126/2431] Added tests for custom payloads. Fixed formatting in tracing tests. --- tests/integration/__init__.py | 4 +- tests/integration/long/test_custom_payload.py | 201 ++++++++++++++++++ tests/integration/standard/test_query.py | 39 ++-- tox.ini | 2 +- 4 files changed, 223 insertions(+), 23 deletions(-) create mode 100644 tests/integration/long/test_custom_payload.py diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3ea397f01e..4c3a3ab0a1 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -203,7 +203,7 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True): if start: log.debug("Starting ccm %s cluster", cluster_name) cluster.start(wait_for_binary_proto=True, wait_other_notice=True) - setup_test_keyspace() + setup_keyspace() CCM_CLUSTER = cluster except Exception: @@ -230,7 +230,7 @@ def teardown_package(): log.warn('Did not find cluster: %s' % cluster_name) -def setup_test_keyspace(): +def setup_keyspace(): # wait for nodes to startup time.sleep(10) diff --git a/tests/integration/long/test_custom_payload.py b/tests/integration/long/test_custom_payload.py new file mode 100644 index 0000000000..f131a5e756 --- /dev/null +++ b/tests/integration/long/test_custom_payload.py @@ -0,0 +1,201 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import unittest2 as unittest +except ImportError: + import unittest + +from cassandra.query import (SimpleStatement, BatchStatement, BatchType) +from cassandra.cluster import Cluster + +from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace + + +def setup_module(): + """ + We need some custom setup for this module. All unit tests in this module + require protocol >=4. We won't bother going through the setup required unless that is the + protocol version we are using. + """ + + # If we aren't at protocol v 4 or greater don't waste time setting anything up, all tests will be skipped + if PROTOCOL_VERSION >= 4: + # Don't start the ccm cluster until we get the custom jvm argument specified + use_singledc(start=False) + ccm_cluster = get_cluster() + # if needed stop CCM cluster + ccm_cluster.stop() + # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back to us + jmv_args = [ + " -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"] + ccm_cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jmv_args) + # wait for nodes to startup + setup_keyspace() + + +def teardown_module(): + """ + The rests of the tests don't need our custom payload query handle so stop the cluster so we + don't impact other tests + """ + + ccm_cluster = get_cluster() + if ccm_cluster is not None: + ccm_cluster.stop() + + +class CustomPayloadTests(unittest.TestCase): + + def setUp(self): + """ + Test is skipped if run with cql version <4 + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest( + "Native protocol 4,0+ is required for custom payloads, currently using %r" + % (PROTOCOL_VERSION,)) + + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + + def tearDown(self): + + self.cluster.shutdown() + + def test_custom_query_basic(self): + """ + Test to validate that custom payloads work with simple queries + + creates a simple query and ensures that custom payloads are passed to C*. A custom + query provider is used with C* so we can validate that same custom payloads are sent back + with the results + + + @since 2.6 + @jira_ticket PYTHON-280 + @expected_result valid custom payloads should be sent and received + + @test_category queries:custom_payload + """ + + # Create a simple query statement a + query = "SELECT * FROM system.local" + statement = SimpleStatement(query) + # Validate that various types of custom payloads are sent and received okay + self.validate_various_custom_payloads(statement=statement) + + def test_custom_query_batching(self): + """ + Test to validate that custom payloads work with batch queries + + creates a batch query and ensures that custom payloads are passed to C*. A custom + query provider is used with C* so we can validate that same custom payloads are sent back + with the results + + + @since 2.6 + @jira_ticket PYTHON-280 + @expected_result valid custom payloads should be sent and received + + @test_category queries:custom_payload + """ + + # Construct Batch Statement + batch = BatchStatement(BatchType.LOGGED) + for i in range(10): + batch.add(SimpleStatement("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)"), (i, i)) + + # Validate that various types of custom payloads are sent and received okay + self.validate_various_custom_payloads(statement=batch) + + def test_custom_query_prepared(self): + """ + Test to validate that custom payloads work with prepared queries + + creates a batch query and ensures that custom payloads are passed to C*. A custom + query provider is used with C* so we can validate that same custom payloads are sent back + with the results + + + @since 2.6 + @jira_ticket PYTHON-280 + @expected_result valid custom payloads should be sent and received + + @test_category queries:custom_payload + """ + + # Construct prepared statement + prepared = self.session.prepare( + """ + INSERT INTO test3rf.test (k, v) VALUES (?, ?) + """) + + bound = prepared.bind((1, None)) + + # Validate that various custom payloads are validated correctly + self.validate_various_custom_payloads(statement=bound) + + def validate_various_custom_payloads(self, statement): + """ + This is a utility method that given a statement will attempt + to submit the statement with various custom payloads. It will + validate that the custom payloads are sent and received correctly. + + @param statement The statement to validate the custom queries in conjunction with + """ + + # Simple key value + custom_payload = {'test': 'test_return'} + self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) + + # no key value + custom_payload = {'': ''} + self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) + + # Space value + custom_payload = {' ': ' '} + self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) + + # Long key value pair + key_value = "x" * 10000 + custom_payload = {key_value: key_value} + self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) + + # Max supported value key pairs according C* binary protocol v4 should be 65534 (unsigned short max value) + for i in range(65534): + custom_payload[str(i)] = str(i) + self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) + + # Add one custom payload to this is too many key value pairs and should fail + custom_payload[str(65535)] = str(65535) + with self.assertRaises(ValueError): + self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) + + def execute_async_validate_custom_payload(self, statement, custom_payload): + """ + This is just a simple method that submits a statement with a payload, and validates + that the custom payload we submitted matches the one that we got back + @param statement The statement to execute + @param custom_payload The custom payload to submit with + """ + + # Submit the statement with our custom payload. Validate the one + # we receive from the server matches + response_future = self.session.execute_async(statement, custom_payload=custom_payload) + response_future.result(timeout=10.0) + returned_custom_payload = response_future.custom_payload + self.assertEqual(custom_payload, returned_custom_payload) \ No newline at end of file diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 7f84f9ab31..6b139c03f1 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -26,7 +26,7 @@ from cassandra.cluster import Cluster from cassandra.policies import HostDistance -from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions +from tests.integration import use_singledc, PROTOCOL_VERSION def setup_module(): @@ -93,40 +93,38 @@ def test_client_ip_in_trace(self): Test to validate that client trace contains client ip information. creates a simple query and ensures that the client trace information is present. This will - only be the case if the c* version is 3.0 or greater + only be the case if the c* version is 2.2 or greater - - @since 3.0 + @since 2.2 @jira_ticket PYTHON-235 - @expected_result client address should be present in C* >= 3, otherwise should be none. + @expected_result client address should be present in C* >= 2.2, otherwise should be none. @test_category tracing -+ """ - #The current version on the trunk doesn't have the version set to 3.0 yet. - #For now we will use the protocol version. Once they update the version on C* trunk - #we can use the C*. See below - #self._cass_version, self._cql_version = get_server_versions() - #if self._cass_version < (3, 0): - # raise unittest.SkipTest("Client IP was not present in trace until C* 3.0") + """ + if PROTOCOL_VERSION < 4: - raise unittest.SkipTest( - "Protocol 4+ is required for client ip tracing, currently testing against %r" - % (PROTOCOL_VERSION,)) + raise unittest.SkipTest( + "Protocol 4+ is required for client ip tracing, currently testing against %r" + % (PROTOCOL_VERSION,)) cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() + # Make simple query with trace enabled query = "SELECT * FROM system.local" statement = SimpleStatement(query) response_future = session.execute_async(statement, trace=True) - response_future.result(10.0) + response_future.result(timeout=10.0) current_host = response_future._current_host.address + # Fetch the client_ip from the trace. - trace = response_future.get_query_trace(2.0) + trace = response_future.get_query_trace(max_wait=2.0) client_ip = trace.client - # Ensure that ip is set for c* >3 - self.assertIsNotNone(client_ip,"Client IP was not set in trace with C* > 3.0") - self.assertEqual(client_ip,current_host,"Client IP from trace did not match the expected value") + + # Ensure that ip is set for c* >2.2 + self.assertIsNotNone(client_ip, "Client IP was not set in trace with C* > 2.2") + self.assertEqual(client_ip, current_host, "Client IP from trace did not match the expected value") + cluster.shutdown() @@ -547,3 +545,4 @@ def test_inherit_first_rk_prepared_param(self): self.assertIsNotNone(batch.routing_key) self.assertEqual(batch.routing_key, self.prepared.bind((1, 0)).routing_key) + diff --git a/tox.ini b/tox.ini index 6c4aa31a27..d694e7ab03 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,8 @@ deps = nose [testenv] deps = {[base]deps} + sure==1.2.3 blist - sure setenv = USE_CASS_EXTERNAL=1 commands = {envpython} setup.py build_ext --inplace nosetests --verbosity=2 tests/unit/ From 8d68dc2c905232a1bf7ab0aff9eb05e6d287735c Mon Sep 17 00:00:00 2001 From: GregBestland Date: Mon, 18 May 2015 09:30:43 -0500 Subject: [PATCH 0127/2431] Documentation fixes --- tests/integration/standard/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 6b139c03f1..cecb07dd51 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -95,7 +95,7 @@ def test_client_ip_in_trace(self): creates a simple query and ensures that the client trace information is present. This will only be the case if the c* version is 2.2 or greater - @since 2.2 + @since 2.6 @jira_ticket PYTHON-235 @expected_result client address should be present in C* >= 2.2, otherwise should be none. From 16f377cbfc21dda65698ecd942db437458db4d1a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 18 May 2015 10:06:56 -0500 Subject: [PATCH 0128/2431] Handle server warnings in protocol v4+ PYTHON-315 --- cassandra/cluster.py | 23 +++++++++++++++++++++++ cassandra/protocol.py | 14 ++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index fe7e3372c4..3bcbe495e9 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1438,6 +1438,8 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False, custom_ future = self.execute_async(query, parameters, trace, custom_payload) try: result = future.result(timeout) + if future.warnings: + log.warning("Query returned warnings from the server. See cassandra.protocol log for details.") finally: if trace: try: @@ -2669,6 +2671,7 @@ class ResponseFuture(object): _metrics = None _paging_state = None _custom_payload = None + _warnings = None def __init__(self, session, message, query, default_timeout=None, metrics=None, prepared_statement=None): self.session = session @@ -2756,6 +2759,24 @@ def has_more_pages(self): """ return self._paging_state is not None + @property + def warnings(self): + """ + Warnings returned from the server, if any. This will only be + set for protocol_version 4+. + + Warnings may be returned for such things as oversized batches, + or too many tombstones in slice queries. + + Ensure the future is complete before trying to access this property + (call :meth:`.result()`, or after callback is invoked). + Otherwise it may throw if the response has not been received. + """ + # TODO: When timers are introduced, just make this wait + if not self._event.is_set(): + raise Exception("warnings cannot be retrieved before ResponseFuture is finalized") + return self._warnings + @property def custom_payload(self): """ @@ -2769,6 +2790,7 @@ def custom_payload(self): :return: :ref:`custom_payload`. """ + # TODO: When timers are introduced, just make this wait if not self._event.is_set(): raise Exception("custom_payload cannot be retrieved before ResponseFuture is finalized") return self._custom_payload @@ -2811,6 +2833,7 @@ def _set_result(self, response): self.query.trace_id = trace_id self._query_trace = QueryTrace(trace_id, self.session) + self._warnings = getattr(response, 'warnings', None) self._custom_payload = getattr(response, 'custom_payload', None) if isinstance(response, ResultMessage): diff --git a/cassandra/protocol.py b/cassandra/protocol.py index d88db589e4..f7c7176f69 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -57,6 +57,7 @@ class InternalError(Exception): COMPRESSED_FLAG = 0x01 TRACING_FLAG = 0x02 CUSTOM_PAYLOAD_FLAG = 0x04 +WARNING_FLAG = 0x08 _message_types_by_name = {} _message_types_by_opcode = {} @@ -74,6 +75,7 @@ class _MessageType(object): tracing = False custom_payload = None + warnings = None def to_binary(self, stream_id, protocol_version, compression=None): flags = 0 @@ -133,6 +135,12 @@ def decode_response(protocol_version, user_type_map, stream_id, flags, opcode, b else: trace_id = None + if flags & WARNING_FLAG: + warnings = read_stringlist(body) + flags ^= WARNING_FLAG + else: + warnings = None + if flags & CUSTOM_PAYLOAD_FLAG: custom_payload = read_bytesmap(body) flags ^= CUSTOM_PAYLOAD_FLAG @@ -147,6 +155,12 @@ def decode_response(protocol_version, user_type_map, stream_id, flags, opcode, b msg.stream_id = stream_id msg.trace_id = trace_id msg.custom_payload = custom_payload + msg.warnings = warnings + + if msg.warnings: + for w in msg.warnings: + log.warning("Server warning: %s", w) + return msg From c8235aade9e75931ba2b0e1e6850df62caf4e5d8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 18 May 2015 16:54:08 -0500 Subject: [PATCH 0129/2431] Remove warn log from Session.execute logging remains in protocol --- cassandra/cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 3bcbe495e9..7d078392eb 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1438,8 +1438,6 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False, custom_ future = self.execute_async(query, parameters, trace, custom_payload) try: result = future.result(timeout) - if future.warnings: - log.warning("Query returned warnings from the server. See cassandra.protocol log for details.") finally: if trace: try: From bc947d0d60b69410d4ed48175c83ec0166d44f76 Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Tue, 19 May 2015 11:38:42 +0800 Subject: [PATCH 0130/2431] Fixed DESCRIBE names for small and tiny ints, added column names --- cassandra/cqlengine/columns.py | 16 +++++++++++++++- cassandra/cqltypes.py | 4 ++-- cassandra/protocol.py | 6 +++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index ae17779590..48a8806c7b 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -377,9 +377,23 @@ def to_database(self, value): return self.validate(value) +class TinyInt(Integer): + """ + Stores an 8-bit signed integer value + """ + db_type = 'tinyint' + + +class SmallInt(Integer): + """ + Stores a 16-bit signed integer value + """ + db_type = 'smallint' + + class BigInt(Integer): """ - Stores a 64-bit signed long value + Stores a 64-bit signed integer value """ db_type = 'bigint' diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 48cbc198c1..6e9306c7cf 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -424,7 +424,7 @@ def deserialize(byts, protocol_version): def serialize(truth, protocol_version): return int8_pack(truth) -class TinyIntType(_CassandraType): +class ByteType(_CassandraType): typename = 'tinyint' @staticmethod @@ -661,7 +661,7 @@ def deserialize(byts, protocol_version): return util.Date(days) -class SmallIntType(_CassandraType): +class ShortType(_CassandraType): typename = 'smallint' @staticmethod diff --git a/cassandra/protocol.py b/cassandra/protocol.py index f7c7176f69..9aab01d298 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -36,7 +36,7 @@ LongType, MapType, SetType, TimeUUIDType, UTF8Type, UUIDType, UserType, TupleType, lookup_casstype, SimpleDateType, - TimeType, TinyIntType, SmallIntType) + TimeType, ByteType, ShortType) from cassandra.policies import WriteType log = logging.getLogger(__name__) @@ -623,8 +623,8 @@ class ResultMessage(_MessageType): 0x0010: InetAddressType, 0x0011: SimpleDateType, 0x0012: TimeType, - 0x0013: SmallIntType, - 0x0014: TinyIntType, + 0x0013: ShortType, + 0x0014: ByteType, 0x0020: ListType, 0x0021: MapType, 0x0022: SetType, From 9b04537f969ce822a3c2ba099f4bbf12e23d72e6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 19 May 2015 08:52:35 -0500 Subject: [PATCH 0131/2431] Update unit tests for corrected small/tiny int types --- tests/unit/test_marshalling.py | 8 ++++---- tests/unit/test_types.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_marshalling.py b/tests/unit/test_marshalling.py index 907cd765b6..60ae33fd6b 100644 --- a/tests/unit/test_marshalling.py +++ b/tests/unit/test_marshalling.py @@ -82,10 +82,10 @@ (b'\x80\x00\x00\x01', 'SimpleDateType', Date(1)), (b'\x7f\xff\xff\xff', 'SimpleDateType', Date('1969-12-31')), (b'\x00\x00\x00\x00\x00\x00\x00\x01', 'TimeType', Time(1)), - (b'\x7f', 'TinyIntType', 127), - (b'\x80', 'TinyIntType', -128), - (b'\x7f\xff', 'SmallIntType', 32767), - (b'\x80\x00', 'SmallIntType', -32768) + (b'\x7f', 'ByteType', 127), + (b'\x80', 'ByteType', -128), + (b'\x7f\xff', 'ShortType', 32767), + (b'\x80\x00', 'ShortType', -32768) ) ordered_map_value = OrderedMapSerializedKey(UTF8Type, 2) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 4386380359..77667705d0 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -26,7 +26,7 @@ from cassandra.cqltypes import (BooleanType, lookup_casstype_simple, lookup_casstype, LongType, DecimalType, SetType, cql_typename, CassandraType, UTF8Type, parse_casstype_args, - SimpleDateType, TimeType, TinyIntType, SmallIntType, + SimpleDateType, TimeType, ByteType, ShortType, EmptyValue, _CassandraType, DateType, int64_pack) from cassandra.encoder import cql_quote from cassandra.protocol import (write_string, read_longstring, write_stringmap, @@ -55,8 +55,8 @@ def test_lookup_casstype_simple(self): self.assertEqual(lookup_casstype_simple('UTF8Type'), cassandra.cqltypes.UTF8Type) self.assertEqual(lookup_casstype_simple('DateType'), cassandra.cqltypes.DateType) self.assertEqual(lookup_casstype_simple('SimpleDateType'), cassandra.cqltypes.SimpleDateType) - self.assertEqual(lookup_casstype_simple('TinyIntType'), cassandra.cqltypes.TinyIntType) - self.assertEqual(lookup_casstype_simple('SmallIntType'), cassandra.cqltypes.SmallIntType) + self.assertEqual(lookup_casstype_simple('ByteType'), cassandra.cqltypes.ByteType) + self.assertEqual(lookup_casstype_simple('ShortType'), cassandra.cqltypes.ShortType) self.assertEqual(lookup_casstype_simple('TimeUUIDType'), cassandra.cqltypes.TimeUUIDType) self.assertEqual(lookup_casstype_simple('TimeType'), cassandra.cqltypes.TimeType) self.assertEqual(lookup_casstype_simple('UUIDType'), cassandra.cqltypes.UUIDType) @@ -89,8 +89,8 @@ def test_lookup_casstype(self): self.assertEqual(lookup_casstype('UTF8Type'), cassandra.cqltypes.UTF8Type) self.assertEqual(lookup_casstype('DateType'), cassandra.cqltypes.DateType) self.assertEqual(lookup_casstype('TimeType'), cassandra.cqltypes.TimeType) - self.assertEqual(lookup_casstype('TinyIntType'), cassandra.cqltypes.TinyIntType) - self.assertEqual(lookup_casstype('SmallIntType'), cassandra.cqltypes.SmallIntType) + self.assertEqual(lookup_casstype('ByteType'), cassandra.cqltypes.ByteType) + self.assertEqual(lookup_casstype('ShortType'), cassandra.cqltypes.ShortType) self.assertEqual(lookup_casstype('TimeUUIDType'), cassandra.cqltypes.TimeUUIDType) self.assertEqual(lookup_casstype('UUIDType'), cassandra.cqltypes.UUIDType) self.assertEqual(lookup_casstype('IntegerType'), cassandra.cqltypes.IntegerType) From c40c1da558e11f6ed4bd3854cdadf7f6cbd7a989 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Wed, 20 May 2015 11:50:17 -0700 Subject: [PATCH 0132/2431] [PYTHON-244] Update docs for test --- .../integration/cqlengine/model/test_model.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/integration/cqlengine/model/test_model.py b/tests/integration/cqlengine/model/test_model.py index c37e088f54..0ac81c35e6 100644 --- a/tests/integration/cqlengine/model/test_model.py +++ b/tests/integration/cqlengine/model/test_model.py @@ -51,6 +51,23 @@ class EqualityModel1(Model): self.assertNotEqual(m0, m1) def test_keywords_as_names(self): + """ + Test for CQL keywords as names + + test_keywords_as_names tests that CQL keywords are properly and automatically quoted in cqlengine. It creates + a keyspace, keyspace, which should be automatically quoted to "keyspace" in CQL. It then creates a table, table, + which should also be automatically quoted to "table". It then verfies that operations can be done on the + "keyspace"."table" which has been created. It also verifies that table alternations work and operations can be + performed on the altered table. + + @since 2.6.0 + @jira_ticket PYTHON-244 + @expected_result Cqlengine should quote CQL keywords properly when creating keyspaces and tables. + + @test_category schema:generation + """ + + # If the keyspace exists, it will not be re-created create_keyspace_simple('keyspace', 1) class table(Model): @@ -58,8 +75,10 @@ class table(Model): select = columns.Integer(primary_key=True) table = columns.Text() - # create should work + # In case the table already exists in keyspace drop_table(table) + + # Create should work sync_table(table) created = table.create(select=0, table='table') @@ -67,7 +86,7 @@ class table(Model): self.assertEqual(created.select, selected.select) self.assertEqual(created.table, selected.table) - # alter should work + # Alter should work class table(Model): __keyspace__ = 'keyspace' select = columns.Integer(primary_key=True) From 8b8993c810e26938e3da92cddca17ae6dfba7548 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 20 May 2015 14:19:10 -0500 Subject: [PATCH 0133/2431] Use a namedtuple for prepared column meta. Improve {maintain, read}ability --- cassandra/protocol.py | 4 +++- cassandra/query.py | 27 ++++++++++----------------- tests/unit/test_parameter_binding.py | 17 +++++++++-------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 9aab01d298..9fbec28520 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import absolute_import # to enable import io from stdlib +from collections import namedtuple import logging import socket from uuid import UUID @@ -49,6 +50,7 @@ class NotSupportedError(Exception): class InternalError(Exception): pass +ColumnMetadata = namedtuple("ColumnMetadata", ['keyspace_name', 'table_name', 'name', 'type']) HEADER_DIRECTION_FROM_CLIENT = 0x00 HEADER_DIRECTION_TO_CLIENT = 0x80 @@ -727,7 +729,7 @@ def recv_prepared_metadata(cls, f, protocol_version, user_type_map): colcfname = read_string(f) colname = read_string(f) coltype = cls.read_type(f, user_type_map) - column_metadata.append((colksname, colcfname, colname, coltype)) + column_metadata.append(ColumnMetadata(colksname, colcfname, colname, coltype)) return column_metadata, pk_indexes @classmethod diff --git a/cassandra/query.py b/cassandra/query.py index 058ba14494..55da7611ee 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -380,15 +380,15 @@ def from_message(cls, query_id, column_metadata, pk_indexes, cluster_metadata, q partition_key_columns = None routing_key_indexes = None - ks_name, table_name, _, _ = column_metadata[0] - ks_meta = cluster_metadata.keyspaces.get(ks_name) + first_col = column_metadata[0] + ks_meta = cluster_metadata.keyspaces.get(first_col.keyspace_name) if ks_meta: - table_meta = ks_meta.tables.get(table_name) + table_meta = ks_meta.tables.get(first_col.table_name) if table_meta: partition_key_columns = table_meta.partition_key # make a map of {column_name: index} for each column in the statement - statement_indexes = dict((c[2], i) for i, c in enumerate(column_metadata)) + statement_indexes = dict((c.name, i) for i, c in enumerate(column_metadata)) # a list of which indexes in the statement correspond to partition key items try: @@ -447,7 +447,7 @@ def __init__(self, prepared_statement, *args, **kwargs): meta = prepared_statement.column_metadata if meta: - self.keyspace = meta[0][0] + self.keyspace = meta[0].keyspace_name Statement.__init__(self, *args, **kwargs) @@ -472,18 +472,16 @@ def bind(self, values): # sort values accordingly for col in col_meta: try: - values.append(dict_values[col[2]]) + values.append(dict_values[col.name]) except KeyError: raise KeyError( 'Column name `%s` not found in bound dict.' % - (col[2])) + (col.name)) # ensure a 1-to-1 dict keys to columns relationship if len(dict_values) != len(col_meta): # find expected columns - columns = set() - for col in col_meta: - columns.add(col[2]) + columns = set(col.name for col in col_meta) # generate error message if len(dict_values) > len(col_meta): @@ -516,17 +514,12 @@ def bind(self, values): if value is None: self.values.append(None) else: - col_type = col_spec[-1] - try: - self.values.append(col_type.serialize(value, proto_version)) + self.values.append(col_spec.type.serialize(value, proto_version)) except (TypeError, struct.error) as exc: - col_name = col_spec[2] - expected_type = col_type actual_type = type(value) - message = ('Received an argument of invalid type for column "%s". ' - 'Expected: %s, Got: %s; (%s)' % (col_name, expected_type, actual_type, exc)) + 'Expected: %s, Got: %s; (%s)' % (col_spec.name, col_spec.type, actual_type, exc)) raise TypeError(message) return self diff --git a/tests/unit/test_parameter_binding.py b/tests/unit/test_parameter_binding.py index 3fce7b9e91..4cc5938fcf 100644 --- a/tests/unit/test_parameter_binding.py +++ b/tests/unit/test_parameter_binding.py @@ -18,8 +18,9 @@ import unittest # noqa from cassandra.encoder import Encoder -from cassandra.query import bind_params, ValueSequence -from cassandra.query import PreparedStatement, BoundStatement +from cassandra.protocol import ColumnMetadata +from cassandra.query import (bind_params, ValueSequence, PreparedStatement, + BoundStatement) from cassandra.cqltypes import Int32Type from cassandra.util import OrderedDict @@ -80,8 +81,8 @@ def test_invalid_argument_type(self): column_family = 'cf1' column_metadata = [ - (keyspace, column_family, 'foo1', Int32Type), - (keyspace, column_family, 'foo2', Int32Type) + ColumnMetadata(keyspace, column_family, 'foo1', Int32Type), + ColumnMetadata(keyspace, column_family, 'foo2', Int32Type) ] prepared_statement = PreparedStatement(column_metadata=column_metadata, @@ -119,8 +120,8 @@ def test_inherit_fetch_size(self): column_family = 'cf1' column_metadata = [ - (keyspace, column_family, 'foo1', Int32Type), - (keyspace, column_family, 'foo2', Int32Type) + ColumnMetadata(keyspace, column_family, 'foo1', Int32Type), + ColumnMetadata(keyspace, column_family, 'foo2', Int32Type) ] prepared_statement = PreparedStatement(column_metadata=column_metadata, @@ -138,8 +139,8 @@ def test_too_few_parameters_for_key(self): column_family = 'cf1' column_metadata = [ - (keyspace, column_family, 'foo1', Int32Type), - (keyspace, column_family, 'foo2', Int32Type) + ColumnMetadata(keyspace, column_family, 'foo1', Int32Type), + ColumnMetadata(keyspace, column_family, 'foo2', Int32Type) ] prepared_statement = PreparedStatement(column_metadata=column_metadata, From 0b8b37ba89930b08a8fb29a6a2f064656dedad6c Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 20 May 2015 16:55:13 -0500 Subject: [PATCH 0134/2431] Distinguish btw NULL and UNSET when binding, proto v4+ PYTHON-317 --- cassandra/protocol.py | 3 + cassandra/query.py | 104 +++++++++++++----- docs/api/cassandra/query.rst | 3 + .../standard/test_prepared_statements.py | 15 ++- 4 files changed, 93 insertions(+), 32 deletions(-) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 9fbec28520..13078b46fd 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -64,6 +64,7 @@ class InternalError(Exception): _message_types_by_name = {} _message_types_by_opcode = {} +_UNSET_VALUE = object() class _RegisterMessageType(type): def __init__(cls, name, bases, dct): @@ -1115,6 +1116,8 @@ def read_value(f): def write_value(f, v): if v is None: write_int(f, -1) + elif v is _UNSET_VALUE: + write_int(f, -2) else: write_int(f, len(v)) f.write(v) diff --git a/cassandra/query.py b/cassandra/query.py index 55da7611ee..374808cd0e 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -24,16 +24,30 @@ import struct import time import six +from six.moves import range from cassandra import ConsistencyLevel, OperationTimedOut from cassandra.util import unix_time_from_uuid1 from cassandra.encoder import Encoder import cassandra.encoder +from cassandra.protocol import _UNSET_VALUE from cassandra.util import OrderedDict import logging log = logging.getLogger(__name__) +UNSET_VALUE = _UNSET_VALUE +""" +Specifies an unset value when binding a prepared statement. + +Unset values are ignored, allowing prepared statements to be used without specify + +See https://issues.apache.org/jira/browse/CASSANDRA-7304 for further details on semantics. + +.. versionadded:: 2.6.0 + +Only valid when using native protocol v4+ +""" NON_ALPHA_REGEX = re.compile('[^a-zA-Z0-9]') START_BADCHAR_REGEX = re.compile('^[^a-zA-Z0-9]*') @@ -350,6 +364,7 @@ class PreparedStatement(object): keyspace = None # change to prepared_keyspace in major release routing_key_indexes = None + _routing_key_index_set = None consistency_level = None serial_consistency_level = None @@ -403,11 +418,16 @@ def from_message(cls, query_id, column_metadata, pk_indexes, cluster_metadata, q def bind(self, values): """ Creates and returns a :class:`BoundStatement` instance using `values`. - The `values` parameter **must** be a sequence, such as a tuple or list, - even if there is only one value to bind. + + See :meth:`BoundStatement.bind` for rules on input ``values``. """ return BoundStatement(self).bind(values) + def is_routing_key_index(self, i): + if self._routing_key_index_set is None: + self._routing_key_index_set = set(self.routing_key_indexes) if self.routing_key_indexes else set() + return i in self._routing_key_index_set + def __str__(self): consistency = ConsistencyLevel.value_to_name.get(self.consistency_level, 'Not Set') return (u'' % @@ -455,64 +475,77 @@ def bind(self, values): """ Binds a sequence of values for the prepared statement parameters and returns this instance. Note that `values` *must* be: + * a sequence, even if you are only binding one value, or * a dict that relates 1-to-1 between dict keys and columns + + .. versionchanged:: 2.6.0 + + :data:`~.UNSET_VALUE` was introduced. These can be bound as positional parameters + in a sequence, or by name in a dict. Additionally, when using protocol v4+: + + * short sequences will be extended to match bind parameters with UNSET_VALUE + * names may be omitted from a dict with UNSET_VALUE implied. + """ if values is None: values = () - col_meta = self.prepared_statement.column_metadata - proto_version = self.prepared_statement.protocol_version + col_meta = self.prepared_statement.column_metadata + col_meta_len = len(col_meta) + value_len = len(values) # special case for binding dicts if isinstance(values, dict): - dict_values = values + unbound_values = values.copy() values = [] # sort values accordingly for col in col_meta: try: - values.append(dict_values[col.name]) + values.append(unbound_values.pop(col.name)) except KeyError: - raise KeyError( - 'Column name `%s` not found in bound dict.' % - (col.name)) + if proto_version >= 4: + values.append(UNSET_VALUE) + else: + raise KeyError( + 'Column name `%s` not found in bound dict.' % + (col.name)) - # ensure a 1-to-1 dict keys to columns relationship - if len(dict_values) != len(col_meta): - # find expected columns - columns = set(col.name for col in col_meta) + if unbound_values: + raise ValueError("Unexpected arguments provided to bind(): %s" % unbound_values.keys()) - # generate error message - if len(dict_values) > len(col_meta): - difference = set(dict_values.keys()).difference(columns) - msg = "Too many arguments provided to bind() (got %d, expected %d). " + \ - "Unexpected keys %s." - else: - difference = set(columns).difference(dict_values.keys()) - msg = "Too few arguments provided to bind() (got %d, expected %d). " + \ - "Expected keys %s." + value_len = len(values) - # exit with error message - msg = msg % (len(values), len(col_meta), difference) - raise ValueError(msg) + if value_len < col_meta_len: + columns = set(col.name for col in col_meta) + difference = set(columns).difference(dict_values.keys()) + raise ValueError("Too few arguments provided to bind() (got %d, expected %d). " + "Missing keys %s." % (value_len, col_meta_len, difference)) - if len(values) > len(col_meta): + if value_len > col_meta_len: raise ValueError( "Too many arguments provided to bind() (got %d, expected %d)" % (len(values), len(col_meta))) - if self.prepared_statement.routing_key_indexes and \ - len(values) < len(self.prepared_statement.routing_key_indexes): + # this is fail-fast for clarity pre-v4. When v4 can be assumed, + # the error will be better reported when UNSET_VALUE is implicitly added. + if proto_version < 4 and self.prepared_statement.routing_key_indexes and \ + value_len < len(self.prepared_statement.routing_key_indexes): raise ValueError( "Too few arguments provided to bind() (got %d, required %d for routing key)" % - (len(values), len(self.prepared_statement.routing_key_indexes))) + (value_len, len(self.prepared_statement.routing_key_indexes))) self.raw_values = values self.values = [] for value, col_spec in zip(values, col_meta): if value is None: self.values.append(None) + elif value is UNSET_VALUE: + if proto_version >= 4: + self._append_unset_value() + else: + raise ValueError("Attempt to bind UNSET_VALUE while using unsuitable protocol version (%d < 4)" % proto_version) else: try: self.values.append(col_spec.type.serialize(value, proto_version)) @@ -522,8 +555,21 @@ def bind(self, values): 'Expected: %s, Got: %s; (%s)' % (col_spec.name, col_spec.type, actual_type, exc)) raise TypeError(message) + if proto_version >= 4: + diff = col_meta_len - len(self.values) + if diff: + for _ in range(diff): + self._append_unset_value() + return self + def _append_unset_value(self): + next_index = len(self.values) + if self.prepared_statement.is_routing_key_index(next_index): + col_meta = self.prepared_statement.column_metadata[next_index] + raise ValueError("Cannot bind UNSET_VALUE as a part of the routing key '%s'" % col_meta.name) + self.values.append(UNSET_VALUE) + @property def routing_key(self): if not self.prepared_statement.routing_key_indexes: diff --git a/docs/api/cassandra/query.rst b/docs/api/cassandra/query.rst index 7e40c18457..55c56cf168 100644 --- a/docs/api/cassandra/query.rst +++ b/docs/api/cassandra/query.rst @@ -23,6 +23,9 @@ .. autoclass:: BoundStatement :members: +.. autodata:: UNSET_VALUE + :annotation: + .. autoclass:: BatchStatement (batch_type=BatchType.LOGGED, retry_policy=None, consistency_level=None) :members: diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index 9240996ce8..e6f6fc8444 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -172,15 +172,24 @@ def test_too_many_bind_values_dicts(self): prepared = session.prepare( """ - INSERT INTO test3rf.test (v) VALUES (?) + INSERT INTO test3rf.test (k, v) VALUES (?, ?) """) self.assertIsInstance(prepared, PreparedStatement) - self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v': 2}) + + # too many values + self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v': 2, 'v2': 3}) + + # right number, but one does not belong + self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v2': 3}) # also catch too few variables with dicts self.assertIsInstance(prepared, PreparedStatement) - self.assertRaises(KeyError, prepared.bind, {}) + if PROTOCOL_VERSION < 4: + self.assertRaises(KeyError, prepared.bind, {}) + else: + # post v4, the driver attempts to use UNSET_VALUE for unspecified keys + self.assertRaises(ValueError, prepared.bind, {}) cluster.shutdown() From f0f3d3191bb58ccfd7613a9bf837d8c947994129 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 21 May 2015 09:08:25 -0500 Subject: [PATCH 0135/2431] Fix typo in Function meta argument_names attribute. --- cassandra/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index ece39cec16..8ede52dc1e 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1043,7 +1043,7 @@ class Function(object): An ordered list of the types for each argument to the function """ - arguemnt_names = None + argument_names = None """ An ordered list of the names of each argument to the function """ From 2302a7848fa1259a6d9a207dbc1c8553db7fd5c6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 21 May 2015 13:42:18 -0500 Subject: [PATCH 0136/2431] Add documentation for connection pool settings PYTHON-304 --- cassandra/cluster.py | 18 ++++++++++++++++++ docs/api/cassandra/cluster.rst | 8 ++++++++ 2 files changed, 26 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 07222d84f0..2a94378a3b 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -660,6 +660,13 @@ def get_min_requests_per_connection(self, host_distance): return self._min_requests_per_connection[host_distance] def set_min_requests_per_connection(self, host_distance, min_requests): + """ + Sets a threshold for concurrent requests per connection, below which + connections will be considered for disposal (down to core connections; + see :meth:`~Cluster.set_core_connections_per_host`). + + Pertains to connection pool management in protocol versions {1,2}. + """ if self.protocol_version >= 3: raise UnsupportedOperation( "Cluster.set_min_requests_per_connection() only has an effect " @@ -670,6 +677,13 @@ def get_max_requests_per_connection(self, host_distance): return self._max_requests_per_connection[host_distance] def set_max_requests_per_connection(self, host_distance, max_requests): + """ + Sets a threshold for concurrent requests per connection, above which new + connections will be created to a host (up to max connections; + see :meth:`~Cluster.set_max_connections_per_host`). + + Pertains to connection pool management in protocol versions {1,2}. + """ if self.protocol_version >= 3: raise UnsupportedOperation( "Cluster.set_max_requests_per_connection() only has an effect " @@ -695,6 +709,10 @@ def set_core_connections_per_host(self, host_distance, core_connections): The default is 2 for :attr:`~HostDistance.LOCAL` and 1 for :attr:`~HostDistance.REMOTE`. + Protocol version 1 and 2 are limited in the number of concurrent + requests they can send per connection. The driver implements connection + pooling to support higher levels of concurrency. + If :attr:`~.Cluster.protocol_version` is set to 3 or higher, this is not supported (there is always one connection per host, unless the host is remote and :attr:`connect_to_remote_hosts` is :const:`False`) diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 52e8469600..c3d71e107e 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -57,6 +57,14 @@ .. automethod:: unregister_listener + .. automethod:: set_max_requests_per_connection + + .. automethod:: get_max_requests_per_connection + + .. automethod:: set_min_requests_per_connection + + .. automethod:: get_min_requests_per_connection + .. automethod:: get_core_connections_per_host .. automethod:: set_core_connections_per_host From a5ca5ee96b213f94df94b8a7d93c55f6389b999a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 21 May 2015 15:15:41 -0500 Subject: [PATCH 0137/2431] Add synchronous refresh mehtods for UDF, UDA --- cassandra/__init__.py | 6 +++--- cassandra/cluster.py | 22 ++++++++++++++++++++++ docs/api/cassandra/cluster.rst | 4 ++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index eff836e7f2..3f61909664 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -135,7 +135,7 @@ def __init__(self, name, type_signature): @property def signature(self): """ - function signatue string in the form 'name([type0[,type1[...]]])' + function signature string in the form 'name([type0[,type1[...]]])' can be used to uniquely identify overloaded function names within a keyspace """ @@ -158,7 +158,7 @@ class UserFunctionDescriptor(SignatureDescriptor): type_signature = None """ - Ordered list of CQL argument type name comprising the type signature + Ordered list of CQL argument type names comprising the type signature """ @@ -174,7 +174,7 @@ class UserAggregateDescriptor(SignatureDescriptor): type_signature = None """ - Ordered list of CQL argument type name comprising the type signature + Ordered list of CQL argument type names comprising the type signature """ diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 07222d84f0..84d60c17de 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1262,6 +1262,28 @@ def refresh_user_type_metadata(self, keyspace, user_type, max_schema_agreement_w if not self.control_connection.refresh_schema(keyspace, usertype=user_type, schema_agreement_wait=max_schema_agreement_wait): raise Exception("User Type metadata was not refreshed. See log for details.") + def refresh_user_function_metadata(self, keyspace, function, max_schema_agreement_wait=None): + """ + Synchronously refresh user defined function metadata. + + ``function`` is a :class:`cassandra.UserFunctionDescriptor`. + + See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior + """ + if not self.control_connection.refresh_schema(keyspace, function=function, schema_agreement_wait=max_schema_agreement_wait): + raise Exception("User Function metadata was not refreshed. See log for details.") + + def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreement_wait=None): + """ + Synchronously refresh user defined aggregate metadata. + + ``aggregate`` is a :class:`cassandra.UserAggregateDescriptor`. + + See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior + """ + if not self.control_connection.refresh_schema(keyspace, aggregate=aggregate, schema_agreement_wait=max_schema_agreement_wait): + raise Exception("User Aggregate metadata was not refreshed. See log for details.") + def refresh_nodes(self): """ Synchronously refresh the node list and token metadata diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 52e8469600..d66d2aa1ea 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -73,6 +73,10 @@ .. automethod:: refresh_user_type_metadata + .. automethod:: refresh_user_function_metadata + + .. automethod:: refresh_user_aggregate_metadata + .. automethod:: refresh_schema .. automethod:: refresh_nodes From 7d628a2b4feae23e63c25807afe4653b6525975e Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 21 May 2015 15:19:07 -0700 Subject: [PATCH 0138/2431] [PYTHON-291] Tests for refresh schema metadata --- tests/integration/standard/test_cluster.py | 21 +- tests/integration/standard/test_metadata.py | 263 +++++++++++++++++++- 2 files changed, 270 insertions(+), 14 deletions(-) diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index 146859ba81..bf6bb1d476 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -248,7 +248,7 @@ def test_refresh_schema(self): original_meta = cluster.metadata.keyspaces # full schema refresh, with wait - cluster.refresh_schema() + cluster.refresh_schema_metadata() self.assertIsNot(original_meta, cluster.metadata.keyspaces) self.assertEqual(original_meta, cluster.metadata.keyspaces) @@ -262,7 +262,7 @@ def test_refresh_schema_keyspace(self): original_system_meta = original_meta['system'] # only refresh one keyspace - cluster.refresh_schema(keyspace='system') + cluster.refresh_keyspace_metadata('system') current_meta = cluster.metadata.keyspaces self.assertIs(original_meta, current_meta) current_system_meta = current_meta['system'] @@ -279,7 +279,7 @@ def test_refresh_schema_table(self): original_system_schema_meta = original_system_meta.tables['schema_columnfamilies'] # only refresh one table - cluster.refresh_schema(keyspace='system', table='schema_columnfamilies') + cluster.refresh_table_metadata('system', 'schema_columnfamilies') current_meta = cluster.metadata.keyspaces current_system_meta = current_meta['system'] current_system_schema_meta = current_system_meta.tables['schema_columnfamilies'] @@ -309,7 +309,7 @@ def test_refresh_schema_type(self): original_type_meta = original_test1rf_meta.user_types[type_name] # only refresh one type - cluster.refresh_schema(keyspace='test1rf', usertype=type_name) + cluster.refresh_user_type_metadata('test1rf', type_name) current_meta = cluster.metadata.keyspaces current_test1rf_meta = current_meta[keyspace_name] current_type_meta = current_test1rf_meta.user_types[type_name] @@ -345,7 +345,7 @@ def test_refresh_schema_no_wait(self): # cluster agreement wait used for refresh original_meta = c.metadata.keyspaces start_time = time.time() - self.assertRaisesRegexp(Exception, r"Schema was not refreshed.*", c.refresh_schema) + self.assertRaisesRegexp(Exception, r"Schema metadata was not refreshed.*", c.refresh_schema_metadata) end_time = time.time() self.assertGreaterEqual(end_time - start_time, agreement_timeout) self.assertIs(original_meta, c.metadata.keyspaces) @@ -353,7 +353,7 @@ def test_refresh_schema_no_wait(self): # refresh wait overrides cluster value original_meta = c.metadata.keyspaces start_time = time.time() - c.refresh_schema(max_schema_agreement_wait=0) + c.refresh_schema_metadata(max_schema_agreement_wait=0) end_time = time.time() self.assertLess(end_time - start_time, agreement_timeout) self.assertIsNot(original_meta, c.metadata.keyspaces) @@ -373,7 +373,7 @@ def test_refresh_schema_no_wait(self): # cluster agreement wait used for refresh original_meta = c.metadata.keyspaces start_time = time.time() - c.refresh_schema() + c.refresh_schema_metadata() end_time = time.time() self.assertLess(end_time - start_time, refresh_threshold) self.assertIsNot(original_meta, c.metadata.keyspaces) @@ -382,7 +382,8 @@ def test_refresh_schema_no_wait(self): # refresh wait overrides cluster value original_meta = c.metadata.keyspaces start_time = time.time() - self.assertRaisesRegexp(Exception, r"Schema was not refreshed.*", c.refresh_schema, max_schema_agreement_wait=agreement_timeout) + self.assertRaisesRegexp(Exception, r"Schema metadata was not refreshed.*", c.refresh_schema_metadata, + max_schema_agreement_wait=agreement_timeout) end_time = time.time() self.assertGreaterEqual(end_time - start_time, agreement_timeout) self.assertIs(original_meta, c.metadata.keyspaces) @@ -566,8 +567,8 @@ def test_pool_management(self): session.execute('USE system_traces') # refresh schema - cluster.refresh_schema() - cluster.refresh_schema(max_schema_agreement_wait=0) + cluster.refresh_schema_metadata() + cluster.refresh_schema_metadata(max_schema_agreement_wait=0) # submit schema refresh future = cluster.submit_schema_refresh() diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 73ffe134ce..69d599f534 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -24,7 +24,8 @@ import sys import traceback -from cassandra import AlreadyExists, OperationTimedOut, SignatureDescriptor +from cassandra import AlreadyExists, OperationTimedOut, SignatureDescriptor, UserFunctionDescriptor, \ + UserAggregateDescriptor from cassandra.cluster import Cluster from cassandra.cqltypes import DoubleType, Int32Type, ListType, UTF8Type, MapType @@ -115,14 +116,14 @@ def check_create_statement(self, tablemeta, original): self.session.execute(recreate) def get_table_metadata(self): - self.cluster.control_connection.refresh_schema() + self.cluster.refresh_table_metadata(self.ksname, self.cfname) return self.cluster.metadata.keyspaces[self.ksname].tables[self.cfname] def test_basic_table_meta_properties(self): create_statement = self.make_create_statement(["a"], [], ["b", "c"]) self.session.execute(create_statement) - self.cluster.control_connection.refresh_schema() + self.cluster.refresh_schema_metadata() meta = self.cluster.metadata self.assertNotEqual(meta.cluster_name, None) @@ -326,6 +327,260 @@ def test_compression_disabled(self): tablemeta = self.get_table_metadata() self.assertIn("compression = {}", tablemeta.export_as_string()) + def test_refresh_schema_metadata(self): + """ + test for synchronously refreshing all cluster metadata + + test_refresh_schema_metadata tests all cluster metadata is refreshed when calling refresh_schema_metadata(). + It creates a second cluster object with schema_event_refresh_window=-1 such that schema refreshes are disabled + for schema change push events. It then alters the cluster, creating a new keyspace, using the first cluster + object, and verifies that the cluster metadata has not changed in the second cluster object. It then calls + refresh_schema_metadata() and verifies that the cluster metadata is updated in the second cluster object. + Similarly, it then proceeds to altering keyspace, table, UDT, UDF, and UDA metadata and subsequently verfies + that these metadata is updated when refresh_schema_metadata() is called. + + @since 2.6.0 + @jira_ticket PYTHON-291 + @expected_result Cluster, keyspace, table, UDT, UDF, and UDA metadata should be refreshed when refresh_schema_metadata() is called. + + @test_category metadata + """ + + cluster2 = Cluster(protocol_version=PROTOCOL_VERSION, schema_event_refresh_window=-1) + cluster2.connect() + + self.assertNotIn("new_keyspace", cluster2.metadata.keyspaces) + + # Cluster metadata modification + self.session.execute("CREATE KEYSPACE new_keyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}") + self.assertNotIn("new_keyspace", cluster2.metadata.keyspaces) + + cluster2.refresh_schema_metadata() + self.assertIn("new_keyspace", cluster2.metadata.keyspaces) + + # Keyspace metadata modification + self.session.execute("ALTER KEYSPACE {0} WITH durable_writes = false".format(self.ksname)) + self.assertTrue(cluster2.metadata.keyspaces[self.ksname].durable_writes) + cluster2.refresh_schema_metadata() + self.assertFalse(cluster2.metadata.keyspaces[self.ksname].durable_writes) + + # Table metadata modification + table_name = "test" + self.session.execute("CREATE TABLE {0}.{1} (a int PRIMARY KEY, b text)".format(self.ksname, table_name)) + cluster2.refresh_schema_metadata() + + self.session.execute("ALTER TABLE {0}.{1} ADD c double".format(self.ksname, table_name)) + self.assertNotIn("c", cluster2.metadata.keyspaces[self.ksname].tables[table_name].columns) + cluster2.refresh_schema_metadata() + self.assertIn("c", cluster2.metadata.keyspaces[self.ksname].tables[table_name].columns) + + if PROTOCOL_VERSION >= 3: + # UDT metadata modification + self.session.execute("CREATE TYPE {0}.user (age int, name text)".format(self.ksname)) + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].user_types, {}) + cluster2.refresh_schema_metadata() + self.assertIn("user", cluster2.metadata.keyspaces[self.ksname].user_types) + + if PROTOCOL_VERSION >= 4: + # UDF metadata modification + self.session.execute("""CREATE FUNCTION {0}.sum_int(key int, val int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 'key + val';""".format(self.ksname)) + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].functions, {}) + cluster2.refresh_schema_metadata() + self.assertIn("sum_int(int,int)", cluster2.metadata.keyspaces[self.ksname].functions) + + # UDA metadata modification + self.session.execute("""CREATE AGGREGATE {0}.sum_agg(int) + SFUNC sum_int + STYPE int + INITCOND 0""" + .format(self.ksname)) + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].aggregates, {}) + cluster2.refresh_schema_metadata() + self.assertIn("sum_agg(int)", cluster2.metadata.keyspaces[self.ksname].aggregates) + + # Cluster metadata modification + self.session.execute("DROP KEYSPACE new_keyspace") + self.assertIn("new_keyspace", cluster2.metadata.keyspaces) + + cluster2.refresh_schema_metadata() + self.assertNotIn("new_keyspace", cluster2.metadata.keyspaces) + + cluster2.shutdown() + + def test_refresh_keyspace_metadata(self): + """ + test for synchronously refreshing keyspace metadata + + test_refresh_keyspace_metadata tests that keyspace metadata is refreshed when calling refresh_keyspace_metadata(). + It creates a second cluster object with schema_event_refresh_window=-1 such that schema refreshes are disabled + for schema change push events. It then alters the keyspace, disabling durable_writes, using the first cluster + object, and verifies that the keyspace metadata has not changed in the second cluster object. Finally, it calls + refresh_keyspace_metadata() and verifies that the keyspace metadata is updated in the second cluster object. + + @since 2.6.0 + @jira_ticket PYTHON-291 + @expected_result Keyspace metadata should be refreshed when refresh_keyspace_metadata() is called. + + @test_category metadata + """ + + cluster2 = Cluster(protocol_version=PROTOCOL_VERSION, schema_event_refresh_window=-1) + cluster2.connect() + + self.assertTrue(cluster2.metadata.keyspaces[self.ksname].durable_writes) + self.session.execute("ALTER KEYSPACE {0} WITH durable_writes = false".format(self.ksname)) + self.assertTrue(cluster2.metadata.keyspaces[self.ksname].durable_writes) + cluster2.refresh_keyspace_metadata(self.ksname) + self.assertFalse(cluster2.metadata.keyspaces[self.ksname].durable_writes) + + cluster2.shutdown() + + def test_refresh_table_metatadata(self): + """ + test for synchronously refreshing table metadata + + test_refresh_table_metatadata tests that table metadata is refreshed when calling test_refresh_table_metatadata(). + It creates a second cluster object with schema_event_refresh_window=-1 such that schema refreshes are disabled + for schema change push events. It then alters the table, adding a new column, using the first cluster + object, and verifies that the table metadata has not changed in the second cluster object. Finally, it calls + test_refresh_table_metatadata() and verifies that the table metadata is updated in the second cluster object. + + @since 2.6.0 + @jira_ticket PYTHON-291 + @expected_result Table metadata should be refreshed when refresh_table_metadata() is called. + + @test_category metadata + """ + + table_name = "test" + self.session.execute("CREATE TABLE {0}.{1} (a int PRIMARY KEY, b text)".format(self.ksname, table_name)) + + cluster2 = Cluster(protocol_version=PROTOCOL_VERSION, schema_event_refresh_window=-1) + cluster2.connect() + + self.assertNotIn("c", cluster2.metadata.keyspaces[self.ksname].tables[table_name].columns) + self.session.execute("ALTER TABLE {0}.{1} ADD c double".format(self.ksname, table_name)) + self.assertNotIn("c", cluster2.metadata.keyspaces[self.ksname].tables[table_name].columns) + + cluster2.refresh_table_metadata(self.ksname, table_name) + self.assertIn("c", cluster2.metadata.keyspaces[self.ksname].tables[table_name].columns) + + cluster2.shutdown() + + def test_refresh_user_type_metadata(self): + """ + test for synchronously refreshing UDT metadata in keyspace + + test_refresh_user_type_metadata tests that UDT metadata in a keyspace is refreshed when calling refresh_user_type_metadata(). + It creates a second cluster object with schema_event_refresh_window=-1 such that schema refreshes are disabled + for schema change push events. It then alters the keyspace, creating a new UDT, using the first cluster + object, and verifies that the UDT metadata has not changed in the second cluster object. Finally, it calls + refresh_user_type_metadata() and verifies that the UDT metadata in the keyspace is updated in the second cluster object. + + @since 2.6.0 + @jira_ticket PYTHON-291 + @expected_result UDT metadata in the keyspace should be refreshed when refresh_user_type_metadata() is called. + + @test_category metadata + """ + + if PROTOCOL_VERSION < 3: + raise unittest.SkipTest("Protocol 3+ is required for UDTs, currently testing against {0}".format(PROTOCOL_VERSION)) + + cluster2 = Cluster(protocol_version=PROTOCOL_VERSION, schema_event_refresh_window=-1) + cluster2.connect() + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].user_types, {}) + self.session.execute("CREATE TYPE {0}.user (age int, name text)".format(self.ksname)) + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].user_types, {}) + + cluster2.refresh_user_type_metadata(self.ksname, "user") + self.assertIn("user", cluster2.metadata.keyspaces[self.ksname].user_types) + + cluster2.shutdown() + + def test_refresh_user_function_metadata(self): + """ + test for synchronously refreshing UDF metadata in keyspace + + test_refresh_user_function_metadata tests that UDF metadata in a keyspace is refreshed when calling + refresh_user_function_metadata(). It creates a second cluster object with schema_event_refresh_window=-1 such + that schema refreshes are disabled for schema change push events. It then alters the keyspace, creating a new + UDF, using the first cluster object, and verifies that the UDF metadata has not changed in the second cluster + object. Finally, it calls refresh_user_function_metadata() and verifies that the UDF metadata in the keyspace + is updated in the second cluster object. + + @since 2.6.0 + @jira_ticket PYTHON-291 + @expected_result UDF metadata in the keyspace should be refreshed when refresh_user_function_metadata() is called. + + @test_category metadata + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol 4+ is required for UDFs, currently testing against {0}".format(PROTOCOL_VERSION)) + + cluster2 = Cluster(protocol_version=PROTOCOL_VERSION, schema_event_refresh_window=-1) + cluster2.connect() + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].functions, {}) + self.session.execute("""CREATE FUNCTION {0}.sum_int(key int, val int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 'key + val';""".format(self.ksname)) + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].functions, {}) + cluster2.refresh_user_function_metadata(self.ksname, UserFunctionDescriptor("sum_int", ["int", "int"])) + self.assertIn("sum_int(int,int)", cluster2.metadata.keyspaces[self.ksname].functions) + + cluster2.shutdown() + + def test_refresh_user_aggregate_metadata(self): + """ + test for synchronously refreshing UDA metadata in keyspace + + test_refresh_user_aggregate_metadata tests that UDA metadata in a keyspace is refreshed when calling + refresh_user_aggregate_metadata(). It creates a second cluster object with schema_event_refresh_window=-1 such + that schema refreshes are disabled for schema change push events. It then alters the keyspace, creating a new + UDA, using the first cluster object, and verifies that the UDA metadata has not changed in the second cluster + object. Finally, it calls refresh_user_aggregate_metadata() and verifies that the UDF metadata in the keyspace + is updated in the second cluster object. + + @since 2.6.0 + @jira_ticket PYTHON-291 + @expected_result UDA metadata in the keyspace should be refreshed when refresh_user_aggregate_metadata() is called. + + @test_category metadata + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol 4+ is required for UDAs, currently testing against {0}".format(PROTOCOL_VERSION)) + + cluster2 = Cluster(protocol_version=PROTOCOL_VERSION, schema_event_refresh_window=-1) + cluster2.connect() + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].aggregates, {}) + self.session.execute("""CREATE FUNCTION {0}.sum_int(key int, val int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE javascript AS 'key + val';""".format(self.ksname)) + + self.session.execute("""CREATE AGGREGATE {0}.sum_agg(int) + SFUNC sum_int + STYPE int + INITCOND 0""" + .format(self.ksname)) + + self.assertEqual(cluster2.metadata.keyspaces[self.ksname].aggregates, {}) + cluster2.refresh_user_aggregate_metadata(self.ksname, UserAggregateDescriptor("sum_agg", ["int"])) + self.assertIn("sum_agg(int)", cluster2.metadata.keyspaces[self.ksname].aggregates) + + cluster2.shutdown() class TestCodeCoverage(unittest.TestCase): @@ -982,7 +1237,7 @@ def test_index_updates(self): self.session.execute("DROP INDEX a_idx") # temporarily synchronously refresh the schema metadata, until CASSANDRA-9391 is merged in - self.cluster.refresh_schema(self.keyspace_name, self.table_name) + self.cluster.refresh_table_metadata(self.keyspace_name, self.table_name) ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] table_meta = ks_meta.tables[self.table_name] From e4d39fbf6f5a8c7d125dc68c4aada665f810bd0b Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 21 May 2015 17:05:40 -0700 Subject: [PATCH 0139/2431] [PYTHON-207] Test for automatic repreparation error --- .../standard/test_prepared_statements.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index 9240996ce8..a2a9beb8a8 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -325,3 +325,31 @@ def test_async_binding_dicts(self): self.assertEqual(results[0].v, None) cluster.shutdown() + + def test_raise_error_on_prepared_statement_execution_dropped_table(self): + """ + test for error in executing prepared statement on a dropped table + + test_raise_error_on_execute_prepared_statement_dropped_table tests that an InvalidRequest is raised when a + prepared statement is executed after its corresponding table is dropped. This happens because if a prepared + statement is invalid, the driver attempts to automatically re-prepare it on a non-existing table. + + @expected_errors InvalidRequest If a prepared statement is executed on a dropped table + + @since 2.6.0 + @jira_ticket PYTHON-207 + @expected_result InvalidRequest error should be raised upon prepared statement execution. + + @test_category prepared_statements + """ + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + session = cluster.connect("test3rf") + + session.execute("CREATE TABLE error_test (k int PRIMARY KEY, v int)") + prepared = session.prepare("SELECT * FROM error_test WHERE k=?") + session.execute("DROP TABLE error_test") + + with self.assertRaises(InvalidRequest): + session.execute(prepared, [0]) + + cluster.shutdown() \ No newline at end of file From 44016c8136a79c82d6ce133f210a02127c22e3af Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 22 May 2015 09:51:08 -0500 Subject: [PATCH 0140/2431] Add FAQ section to docs PYTHON-115 --- docs/cqlengine/faq.rst | 8 ++-- docs/faq.rst | 83 ++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 6 +++ 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 docs/faq.rst diff --git a/docs/cqlengine/faq.rst b/docs/cqlengine/faq.rst index 6342538de4..dcaefae22c 100644 --- a/docs/cqlengine/faq.rst +++ b/docs/cqlengine/faq.rst @@ -2,15 +2,15 @@ Frequently Asked Questions ========================== -Q: Why don't updates work correctly on models instantiated as Model(field=value, field2=value2)? +Why don't updates work correctly on models instantiated as Model(field=value, field2=value2)? ------------------------------------------------------------------------------------------------ -A: The recommended way to create new rows is with the models .create method. The values passed into a model's init method are interpreted by the model as the values as they were read from a row. This allows the model to "know" which rows have changed since the row was read out of cassandra, and create suitable update statements. +The recommended way to create new rows is with the models .create method. The values passed into a model's init method are interpreted by the model as the values as they were read from a row. This allows the model to "know" which rows have changed since the row was read out of cassandra, and create suitable update statements. -Q: How to preserve ordering in batch query? +How to preserve ordering in batch query? ------------------------------------------- -A: Statement Ordering is not supported by CQL3 batches. Therefore, +Statement Ordering is not supported by CQL3 batches. Therefore, once cassandra needs resolving conflict(Updating the same column in one batch), The algorithm below would be used. diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 0000000000..56cb648a24 --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,83 @@ +Frequently Asked Questions +========================== + +See also :doc:`cqlengine FAQ ` + +Why do connections or IO operations timeout in my WSGI application? +------------------------------------------------------------------- +Depending on your application process model, it may be forking after driver Session is created. Most IO reactors do not handle this, and problems will manifest as timeouts. + +To avoid this, make sure to create sessions per process, after the fork. Using uWSGI and Flask for example: + +.. code-block:: python + + from flask import Flask + from uwsgidecorators import postfork + from cassandra.cluster import Cluster + + session = None + prepared = None + + @postfork + def connect(): + global session, prepared + session = Cluster().connect() + prepared = session.prepare("SELECT release_version FROM system.local WHERE key=?") + + app = Flask(__name__) + + @app.route('/') + def server_version(): + row = session.execute(prepared, ('local',))[0] + return row.release_version + +uWSGI provides a ``postfork`` hook you can use to create sessions and prepared statements after the child process forks. + +How do I trace a request? +------------------------- +Request tracing can be turned on for any request by setting ``trace=True`` in :meth:`.Session.execute_async`. View the results by waiting on the future, then :meth:`.ResponseFuture.get_query_trace`. +Since tracing is done asynchronously to the request, this method polls until the trace is complete before querying data. + +.. code-block:: python + + >>> future = session.execute_async("SELECT * FROM system.local", trace=True) + >>> result = future.result() + >>> trace = future.get_query_trace() + >>> for e in trace.events: + >>> print e.source_elapsed, e.description + + 0:00:00.000077 Parsing select * from system.local + 0:00:00.000153 Preparing statement + 0:00:00.000309 Computing ranges to query + 0:00:00.000368 Submitting range requests on 1 ranges with a concurrency of 1 (279.77142 rows per range expected) + 0:00:00.000422 Submitted 1 concurrent range requests covering 1 ranges + 0:00:00.000480 Executing seq scan across 1 sstables for (min(-9223372036854775808), min(-9223372036854775808)) + 0:00:00.000669 Read 1 live and 0 tombstone cells + 0:00:00.000755 Scanned 1 rows and matched 1 + +``trace`` is a :class:`QueryTrace` object. + +How do I determine the replicas for a query? +---------------------------------------------- +With prepared statements, the replicas are obtained by ``routing_key``, based on current cluster token metadata: + +.. code-block:: python + + >>> prepared = session.prepare("SELECT * FROM example.t WHERE key=?") + >>> bound = prepared.bind((1,)) + >>> replicas = cluster.metadata.get_replicas(bound.keyspace, bound.routing_key) + >>> for h in replicas: + >>> print h.address + 127.0.0.1 + 127.0.0.2 + +``replicas`` is a list of :class:`Host` objects. + +How does the driver manage request retries? +------------------------------------------- +By default, retries are managed by the :attr:`.Cluster.default_retry_policy` set on the session Cluster. It can also +be specialized per statement by setting :attr:`.Statement.retry_policy`. + +Retries are presently attempted on the same coordinator, but this may change in the future. + +Please see :class:`.policies.RetryPolicy` for further details. diff --git a/docs/index.rst b/docs/index.rst index e28b00685b..0b89c369da 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,9 @@ Contents :doc:`security` An overview of the security features of the driver. +:doc:`faq` + A collection of Frequently Asked Questions + .. toctree:: :hidden: @@ -51,9 +54,12 @@ Contents security user_defined_types object_mapper + faq Getting Help ------------ +Visit the :doc:`FAQ section ` in this documentation. + Please send questions to the `mailing list `_. Alternatively, you can use IRC. Connect to the #datastax-drivers channel on irc.freenode.net. From 88e927fd299e22d669d1906bbbb53aad2a09c350 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 22 May 2015 13:03:04 -0500 Subject: [PATCH 0141/2431] Remove dead conditional in BoundStatement.bind --- cassandra/query.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cassandra/query.py b/cassandra/query.py index 374808cd0e..6709cbeff6 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -392,7 +392,6 @@ def from_message(cls, query_id, column_metadata, pk_indexes, cluster_metadata, q if pk_indexes: routing_key_indexes = pk_indexes else: - partition_key_columns = None routing_key_indexes = None first_col = column_metadata[0] @@ -512,16 +511,10 @@ def bind(self, values): 'Column name `%s` not found in bound dict.' % (col.name)) - if unbound_values: - raise ValueError("Unexpected arguments provided to bind(): %s" % unbound_values.keys()) - value_len = len(values) - if value_len < col_meta_len: - columns = set(col.name for col in col_meta) - difference = set(columns).difference(dict_values.keys()) - raise ValueError("Too few arguments provided to bind() (got %d, expected %d). " - "Missing keys %s." % (value_len, col_meta_len, difference)) + if unbound_values: + raise ValueError("Unexpected arguments provided to bind(): %s" % unbound_values.keys()) if value_len > col_meta_len: raise ValueError( From effd1a9fc4c0186b1ca6a2677a0a1d44982eef11 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 22 May 2015 13:03:35 -0500 Subject: [PATCH 0142/2431] Unit tests for binding NULL vs UNSET in protocol v4 --- tests/unit/test_parameter_binding.py | 139 +++++++++++++++++++-------- 1 file changed, 100 insertions(+), 39 deletions(-) diff --git a/tests/unit/test_parameter_binding.py b/tests/unit/test_parameter_binding.py index 4cc5938fcf..6b84260279 100644 --- a/tests/unit/test_parameter_binding.py +++ b/tests/unit/test_parameter_binding.py @@ -20,7 +20,7 @@ from cassandra.encoder import Encoder from cassandra.protocol import ColumnMetadata from cassandra.query import (bind_params, ValueSequence, PreparedStatement, - BoundStatement) + BoundStatement, UNSET_VALUE) from cassandra.cqltypes import Int32Type from cassandra.util import OrderedDict @@ -74,42 +74,42 @@ def test_float_precision(self): self.assertEqual(float(bind_params("%s", (f,), Encoder())), f) -class BoundStatementTestCase(unittest.TestCase): +class BoundStatementTestV1(unittest.TestCase): - def test_invalid_argument_type(self): - keyspace = 'keyspace1' - column_family = 'cf1' + protocol_version=1 - column_metadata = [ - ColumnMetadata(keyspace, column_family, 'foo1', Int32Type), - ColumnMetadata(keyspace, column_family, 'foo2', Int32Type) - ] - - prepared_statement = PreparedStatement(column_metadata=column_metadata, - query_id=None, - routing_key_indexes=[], - query=None, - keyspace=keyspace, - protocol_version=2) - bound_statement = BoundStatement(prepared_statement=prepared_statement) - - values = ['nonint', 1] + @classmethod + def setUpClass(cls): + cls.prepared = PreparedStatement(column_metadata=[ + ColumnMetadata('keyspace', 'cf', 'rk0', Int32Type), + ColumnMetadata('keyspace', 'cf', 'rk1', Int32Type), + ColumnMetadata('keyspace', 'cf', 'ck0', Int32Type), + ColumnMetadata('keyspace', 'cf', 'v0', Int32Type) + ], + query_id=None, + routing_key_indexes=[1, 0], + query=None, + keyspace='keyspace', + protocol_version=cls.protocol_version) + cls.bound = BoundStatement(prepared_statement=cls.prepared) + def test_invalid_argument_type(self): + values = (0, 0, 0, 'string not int') try: - bound_statement.bind(values) + self.bound.bind(values) except TypeError as e: - self.assertIn('foo1', str(e)) + self.assertIn('v0', str(e)) self.assertIn('Int32Type', str(e)) self.assertIn('str', str(e)) else: self.fail('Passed invalid type but exception was not thrown') - values = [1, ['1', '2']] + values = (['1', '2'], 0, 0, 0) try: - bound_statement.bind(values) + self.bound.bind(values) except TypeError as e: - self.assertIn('foo2', str(e)) + self.assertIn('rk0', str(e)) self.assertIn('Int32Type', str(e)) self.assertIn('list', str(e)) else: @@ -129,28 +129,89 @@ def test_inherit_fetch_size(self): routing_key_indexes=[], query=None, keyspace=keyspace, - protocol_version=2) + protocol_version=self.protocol_version) prepared_statement.fetch_size = 1234 bound_statement = BoundStatement(prepared_statement=prepared_statement) self.assertEqual(1234, bound_statement.fetch_size) - def test_too_few_parameters_for_key(self): - keyspace = 'keyspace1' - column_family = 'cf1' + def test_too_few_parameters_for_routing_key(self): + self.assertRaises(ValueError, self.prepared.bind, (1,)) - column_metadata = [ - ColumnMetadata(keyspace, column_family, 'foo1', Int32Type), - ColumnMetadata(keyspace, column_family, 'foo2', Int32Type) - ] + bound = self.prepared.bind((1, 2)) + self.assertEqual(bound.keyspace, 'keyspace') - prepared_statement = PreparedStatement(column_metadata=column_metadata, + def test_dict_missing_routing_key(self): + self.assertRaises(KeyError, self.bound.bind, {'rk0': 0, 'ck0': 0, 'v0': 0}) + self.assertRaises(KeyError, self.bound.bind, {'rk1': 0, 'ck0': 0, 'v0': 0}) + + def test_missing_value(self): + self.assertRaises(KeyError, self.bound.bind, {'rk0': 0, 'rk1': 0, 'ck0': 0}) + # legacy behavior: short parameters are allowed through, errored by server + #self.assertRaises(ValueError, self.bound.bind, (0, 0, 0)) + + def test_dict_extra_value(self): + self.assertRaises(ValueError, self.bound.bind, {'rk0': 0, 'rk1': 0, 'ck0': 0, 'v0': 0, 'should_not_be_here': 123}) + self.assertRaises(ValueError, self.bound.bind, (0, 0, 0, 0, 123)) + + def test_values_none(self): + # should have values + self.assertRaises(ValueError, self.bound.bind, None) + + # prepared statement with no values + prepared_statement = PreparedStatement(column_metadata=[], query_id=None, - routing_key_indexes=[0, 1], + routing_key_indexes=[], query=None, - keyspace=keyspace, - protocol_version=2) + keyspace='whatever', + protocol_version=self.protocol_version) + bound = prepared_statement.bind(None) + self.assertListEqual(bound.values, []) + + def test_bind_none(self): + self.bound.bind({'rk0': 0, 'rk1': 0, 'ck0': 0, 'v0': None}) + self.assertEqual(self.bound.values[-1], None) + + old_values = self.bound.values + self.bound.bind((0, 0, 0, None)) + self.assertIsNot(self.bound.values, old_values) + self.assertEqual(self.bound.values[-1], None) + + def test_unset_value(self): + self.assertRaises(ValueError, self.bound.bind, {'rk0': 0, 'rk1': 0, 'ck0': 0, 'v0': UNSET_VALUE}) + self.assertRaises(ValueError, self.bound.bind, (0, 0, 0, UNSET_VALUE)) + + +class BoundStatementTestV2(BoundStatementTestV1): + protocol_version=2 + + +class BoundStatementTestV3(BoundStatementTestV1): + protocol_version=3 + + +class BoundStatementTestV4(BoundStatementTestV1): + protocol_version=4 + + def test_dict_missing_routing_key(self): + # in v4 it implicitly binds UNSET_VALUE for missing items, + # UNSET_VALUE is ValueError for routing keys + self.assertRaises(ValueError, self.bound.bind, {'rk0': 0, 'ck0': 0, 'v0': 0}) + self.assertRaises(ValueError, self.bound.bind, {'rk1': 0, 'ck0': 0, 'v0': 0}) + + def test_missing_value(self): + # in v4 missing values are UNSET_VALUE + self.bound.bind({'rk0': 0, 'rk1': 0, 'ck0': 0}) + self.assertEqual(self.bound.values[-1], UNSET_VALUE) + + old_values = self.bound.values + self.bound.bind((0, 0, 0)) + self.assertIsNot(self.bound.values, old_values) + self.assertEqual(self.bound.values[-1], UNSET_VALUE) - self.assertRaises(ValueError, prepared_statement.bind, (1,)) + def test_unset_value(self): + self.bound.bind({'rk0': 0, 'rk1': 0, 'ck0': 0, 'v0': UNSET_VALUE}) + self.assertEqual(self.bound.values[-1], UNSET_VALUE) - bound = prepared_statement.bind((1, 2)) - self.assertEqual(bound.keyspace, keyspace) + old_values = self.bound.values + self.bound.bind((0, 0, 0, UNSET_VALUE)) + self.assertEqual(self.bound.values[-1], UNSET_VALUE) From 43f1ea748ceb52d4aebfcb12f18c43583c2a46ac Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 22 May 2015 14:12:25 -0500 Subject: [PATCH 0143/2431] Integration test for binding UNSET values in v4+ PYTHON-317 --- .../standard/test_prepared_statements.py | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index e6f6fc8444..9d44a383ba 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -22,7 +22,7 @@ from cassandra import InvalidRequest from cassandra.cluster import Cluster -from cassandra.query import PreparedStatement +from cassandra.query import PreparedStatement, UNSET_VALUE def setup_module(): @@ -222,6 +222,54 @@ def test_none_values(self): cluster.shutdown() + def test_unset_values(self): + """ + Test to validate that UNSET_VALUEs are bound, and have the expected effect + + Prepare a statement and insert all values. Then follow with execute excluding + parameters. Verify that the original values are unaffected. + + @since 2.6.0 + + @jira_ticket PYTHON-317 + @expected_result UNSET_VALUE is implicitly added to bind parameters, and properly encoded, leving unset values unaffected. + + @test_category prepared_statements:binding + """ + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Binding UNSET values is not supported in protocol version < 4") + + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + session = cluster.connect() + + # table with at least two values so one can be used as a marker + session.execute("CREATE TABLE IF NOT EXISTS test1rf.test_unset_values (k int PRIMARY KEY, v0 int, v1 int)") + insert = session.prepare( "INSERT INTO test1rf.test_unset_values (k, v0, v1) VALUES (?, ?, ?)") + select = session.prepare( "SELECT * FROM test1rf.test_unset_values WHERE k=?") + + bind_expected = [ + # initial condition + ((0, 0, 0), (0, 0, 0)), + # unset implicit + ((0, 1,), (0, 1, 0)), + ({'k': 0, 'v0': 2}, (0, 2, 0)), + ({'k': 0, 'v1': 1}, (0, 2, 1)), + # unset explicit + ((0, 3, UNSET_VALUE), (0, 3, 1)), + ((0, UNSET_VALUE, 2), (0, 3, 2)), + ({'k': 0, 'v0': 4, 'v1': UNSET_VALUE}, (0, 4, 2)), + ({'k': 0, 'v0': UNSET_VALUE, 'v1': 3}, (0, 4, 3)), + # nulls still work + ((0, None, None), (0, None, None)), + ] + + for params, expected in bind_expected: + session.execute(insert, params) + results = session.execute(select, (0,)) + self.assertEqual(results[0], expected) + + cluster.shutdown() + def test_no_meta(self): cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() From 5bd44a97c32d587412389fa1148ff01fba615951 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 22 May 2015 14:21:44 -0500 Subject: [PATCH 0144/2431] Fix test for wrong dict values passed to bind PYTHON-317 --- tests/integration/standard/test_prepared_statements.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index 9d44a383ba..8c50d7b747 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -163,7 +163,7 @@ def test_too_many_bind_values(self): def test_too_many_bind_values_dicts(self): """ - Ensure a ValueError is thrown when attempting to bind too many variables + Ensure an error is thrown when attempting to bind the wrong values with dict bindings """ @@ -181,7 +181,12 @@ def test_too_many_bind_values_dicts(self): self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v': 2, 'v2': 3}) # right number, but one does not belong - self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v2': 3}) + if PROTOCOL_VERSION < 4: + # pre v4, the driver bails with key error when 'v' is found missing + self.assertRaises(KeyError, prepared.bind, {'k': 1, 'v2': 3}) + else: + # post v4, the driver uses UNSET_VALUE for 'v' and bails when 'v2' is unbound + self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v2': 3}) # also catch too few variables with dicts self.assertIsInstance(prepared, PreparedStatement) From 09505e65974af138763f46ac0bfdf24fa738a040 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Thu, 21 May 2015 17:51:43 -0500 Subject: [PATCH 0145/2431] [PYTHON-238] Added tests for protocol v4 Exceptions, and MISC test cleanup --- tests/integration/long/test_failure_types.py | 279 +++++++++++++++++++ tests/integration/standard/test_query.py | 8 +- 2 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 tests/integration/long/test_failure_types.py diff --git a/tests/integration/long/test_failure_types.py b/tests/integration/long/test_failure_types.py new file mode 100644 index 0000000000..a927dc95f9 --- /dev/null +++ b/tests/integration/long/test_failure_types.py @@ -0,0 +1,279 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import unittest2 as unittest +except ImportError: + import unittest + + +from cassandra.cluster import Cluster +from cassandra import ConsistencyLevel +from cassandra import WriteFailure, ReadFailure, FunctionFailure +from cassandra.concurrent import execute_concurrent_with_args +from cassandra.query import SimpleStatement +from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace + + +def setup_module(): + """ + We need some custom setup for this module. All unit tests in this module + require protocol >=4. We won't bother going through the setup required unless that is the + protocol version we are using. + """ + + # If we aren't at protocol v 4 or greater don't waste time setting anything up, all tests will be skipped + if PROTOCOL_VERSION >= 4: + use_singledc(start=False) + ccm_cluster = get_cluster() + ccm_cluster.stop() + config_options = {'tombstone_failure_threshold': 2000, 'tombstone_warn_threshold': 1000} + ccm_cluster.set_configuration_options(config_options) + ccm_cluster.start(wait_for_binary_proto=True, wait_other_notice=True) + setup_keyspace() + + +def teardown_module(): + """ + The rest of the tests don't need custom tombstones + reset the config options so as to not mess with other tests. + """ + if PROTOCOL_VERSION >= 4: + ccm_cluster = get_cluster() + config_options = {} + ccm_cluster.set_configuration_options(config_options) + if ccm_cluster is not None: + ccm_cluster.stop() + + +class ClientExceptionTests(unittest.TestCase): + + def setUp(self): + """ + Test is skipped if run with native protocol version <4 + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest( + "Native protocol 4,0+ is required for custom payloads, currently using %r" + % (PROTOCOL_VERSION,)) + + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + self.nodes_currently_failing = [] + self.node1, self.node2, self.node3 = get_cluster().nodes.values() + + def tearDown(self): + + self.cluster.shutdown() + failing_nodes = [] + + # Restart the nodes to fully functional again + self.setFailingNodes(failing_nodes, "testksfail") + + def setFailingNodes(self, failing_nodes, keyspace): + """ + This method will take in a set of failing nodes, and toggle all of the nodes in the provided list to fail + writes. + @param failing_nodes A definitive list of nodes that should fail writes + @param keyspace The keyspace to enable failures on + + """ + + # Ensure all of the nodes on the list have failures enabled + for node in failing_nodes: + if node not in self.nodes_currently_failing: + node.stop(wait_other_notice=True, gently=False) + node.start(jvm_args=[" -Dcassandra.test.fail_writes_ks=" + keyspace], wait_for_binary_proto=True, + wait_other_notice=True) + self.nodes_currently_failing.append(node) + + # Ensure all nodes not on the list, but that are currently set to failing are enabled + for node in self.nodes_currently_failing: + if node not in failing_nodes: + node.stop(wait_other_notice=True, gently=False) + node.start(wait_for_binary_proto=True, wait_other_notice=True) + self.nodes_currently_failing.remove(node) + + def _perform_cql_statement(self, text, consistency_level, expected_exception): + """ + Simple helper method to preform cql statements and check for expected exception + @param text CQl statement to execute + @param consistency_level Consistency level at which it is to be executed + @param expected_exception Exception expected to be throw or none + """ + statement = SimpleStatement(text) + statement.consistency_level = consistency_level + + if expected_exception is None: + self.session.execute(statement) + else: + with self.assertRaises(expected_exception): + self.session.execute(statement) + + def test_write_failures_from_coordinator(self): + """ + Test to validate that write failures from the coordinator are surfaced appropriately. + + test_write_failures_from_coordinator Enable write failures on the various nodes using a custom jvm flag, + cassandra.test.fail_writes_ks. This will cause writes to fail on that specific node. Depending on the replication + factor of the keyspace, and the consistency level, we will expect the coordinator to send WriteFailure, or not. + + + @since 2.6.0 + @jira_ticket PYTHON-238 + @expected_result Appropriate write failures from the coordinator + + @test_category queries:basic + """ + + # Setup temporary keyspace. + self._perform_cql_statement( + """ + CREATE KEYSPACE testksfail + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + # create table + self._perform_cql_statement( + """ + CREATE TABLE testksfail.test ( + k int PRIMARY KEY, + v int ) + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + # Disable one node + failing_nodes = [self.node1] + self.setFailingNodes(failing_nodes, "testksfail") + + # With one node disabled we would expect a write failure with ConsistencyLevel of all + self._perform_cql_statement( + """ + INSERT INTO testksfail.test (k, v) VALUES (1, 0 ) + """, consistency_level=ConsistencyLevel.ALL, expected_exception=WriteFailure) + + # We have two nodes left so a write with consistency level of QUORUM should complete as expected + self._perform_cql_statement( + """ + INSERT INTO testksfail.test (k, v) VALUES (1, 0 ) + """, consistency_level=ConsistencyLevel.QUORUM, expected_exception=None) + + failing_nodes = [] + + # Restart the nodes to fully functional again + self.setFailingNodes(failing_nodes, "testksfail") + + # Drop temporary keyspace + self._perform_cql_statement( + """ + DROP KEYSPACE testksfail + """, consistency_level=ConsistencyLevel.ANY, expected_exception=None) + + def test_tombstone_overflow_read_failure(self): + """ + Test to validate that a ReadFailure is returned from the node when a specified threshold of tombstombs is + reached. + + test_tombstomb_overflow_read_failure First sets the tombstone failure threshold down to a level that allows it + to be more easily encountered. We then create some wide rows and ensure they are deleted appropriately. This + produces the correct amount of tombstombs. Upon making a simple query we expect to get a read failure back + from the coordinator. + + + @since 2.6.0 + @jira_ticket PYTHON-238 + @expected_result Appropriate write failures from the coordinator + + @test_category queries:basic + """ + + # Setup table for "wide row" + self._perform_cql_statement( + """ + CREATE TABLE test3rf.test2 ( + k int, + v0 int, + v1 int, PRIMARY KEY (k,v0)) + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + statement = self.session.prepare("INSERT INTO test3rf.test2 (k, v0,v1) VALUES (1,?,1)") + parameters = [(x,) for x in range(3000)] + execute_concurrent_with_args(self.session, statement, parameters, concurrency=50) + + statement = self.session.prepare("DELETE v1 FROM test3rf.test2 WHERE k = 1 AND v0 =?") + parameters = [(x,) for x in range(2001)] + execute_concurrent_with_args(self.session, statement, parameters, concurrency=50) + + self._perform_cql_statement( + """ + SELECT * FROM test3rf.test2 WHERE k = 1 + """, consistency_level=ConsistencyLevel.ALL, expected_exception=ReadFailure) + + self._perform_cql_statement( + """ + DROP TABLE test3rf.test2; + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + def test_user_function_failure(self): + """ + Test to validate that exceptions in user defined function are correctly surfaced by the driver to us. + + test_user_function_failure First creates a table to use for testing. Then creates a function that will throw an + exception when invoked. It then invokes the function and expects a FunctionException. Finally it preforms + cleanup operations. + + @since 2.6.0 + @jira_ticket PYTHON-238 + @expected_result Function failures when UDF throws exception + + @test_category queries:basic + """ + + # create UDF that throws an exception + self._perform_cql_statement( + """ + CREATE FUNCTION test3rf.test_failure(d double) + RETURNS NULL ON NULL INPUT + RETURNS double + LANGUAGE java AS 'throw new RuntimeException("failure");'; + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + # Create test table + self._perform_cql_statement( + """ + CREATE TABLE test3rf.d (k int PRIMARY KEY , d double); + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + # Insert some values + self._perform_cql_statement( + """ + INSERT INTO test3rf.d (k,d) VALUES (0, 5.12); + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + # Run the function expect a function failure exception + self._perform_cql_statement( + """ + SELECT test_failure(d) FROM test3rf.d WHERE k = 0; + """, consistency_level=ConsistencyLevel.ALL, expected_exception=FunctionFailure) + + self._perform_cql_statement( + """ + DROP FUNCTION test3rf.test_failure; + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + self._perform_cql_statement( + """ + DROP TABLE test3rf.d; + """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 97de830b85..dc98505b87 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -28,6 +28,8 @@ from tests.integration import use_singledc, PROTOCOL_VERSION +import re + def setup_module(): use_singledc() @@ -136,15 +138,17 @@ def test_client_ip_in_trace(self): statement = SimpleStatement(query) response_future = session.execute_async(statement, trace=True) response_future.result(timeout=10.0) - current_host = response_future._current_host.address # Fetch the client_ip from the trace. trace = response_future.get_query_trace(max_wait=2.0) client_ip = trace.client + # Ip address should be in the local_host range + pat = re.compile("127.0.0.\d{1,3}") + # Ensure that ip is set self.assertIsNotNone(client_ip, "Client IP was not set in trace with C* >= 2.2") - self.assertEqual(client_ip, current_host, "Client IP from trace did not match the expected value") + self.assertTrue(pat.match(client_ip), "Client IP from trace did not match the expected value") cluster.shutdown() From 7aa8f3037538b786814ec870ce8282e7f3c534db Mon Sep 17 00:00:00 2001 From: Tom Lin Date: Mon, 25 May 2015 16:11:35 +0800 Subject: [PATCH 0146/2431] Check if max_attempts is None --- cassandra/policies.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index ce879abb5e..d4d8914c8b 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -509,14 +509,16 @@ def __init__(self, delay, max_attempts=64): """ if delay < 0: raise ValueError("delay must not be negative") - if max_attempts < 0: + if max_attempts is not None and max_attempts < 0: raise ValueError("max_attempts must not be negative") self.delay = delay self.max_attempts = max_attempts def new_schedule(self): - return repeat(self.delay, self.max_attempts) + if self.max_attempts: + return repeat(self.delay, self.max_attempts) + return repeat(self.delay) class ExponentialReconnectionPolicy(ReconnectionPolicy): From b43a947d4a7a88e395939b370bd616f396dee6a1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 19 May 2015 16:27:48 -0500 Subject: [PATCH 0147/2431] cqle: introduce 'date' and 'time' column types PYTHON-245 --- cassandra/cqlengine/columns.py | 63 +++++++++++++++--------- cassandra/cqltypes.py | 28 ++++++----- cassandra/util.py | 4 +- docs/api/cassandra/cqlengine/columns.rst | 8 ++- 4 files changed, 63 insertions(+), 40 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index 15c05532fc..9edf0d6588 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -19,9 +19,8 @@ import six import warnings -from cassandra.cqltypes import DateType -from cassandra.encoder import cql_quote - +from cassandra import util +from cassandra.cqltypes import DateType, SimpleDateType from cassandra.cqlengine import ValidationError log = logging.getLogger(__name__) @@ -361,6 +360,10 @@ def to_database(self, value): class TinyInt(Integer): """ Stores an 8-bit signed integer value + + .. versionadded:: 2.6.0 + + requires C* 2.2+ and protocol v4+ """ db_type = 'tinyint' @@ -368,6 +371,10 @@ class TinyInt(Integer): class SmallInt(Integer): """ Stores a 16-bit signed integer value + + .. versionadded:: 2.6.0 + + requires C* 2.2+ and protocol v4+ """ db_type = 'smallint' @@ -466,35 +473,43 @@ def to_database(self, value): class Date(Column): """ - *Note: this type is overloaded, and will likely be changed or removed to accommodate distinct date type - in a future version* + Stores a simple date, with no time-of-day + + .. versionchanged:: 2.6.0 + + removed overload of Date and DateTime. DateTime is a drop-in replacement for legacy models - Stores a date value, with no time-of-day + requires C* 2.2+ and protocol v4+ """ - db_type = 'timestamp' + db_type = 'date' - def to_python(self, value): + def to_database(self, value): + value = super(Date, self).to_database(value) if value is None: return - if isinstance(value, datetime): - return value.date() - elif isinstance(value, date): - return value - try: - return datetime.utcfromtimestamp(value).date() - except TypeError: - return datetime.utcfromtimestamp(DateType.deserialize(value)).date() + + # need to translate to int version because some dates are not representable in + # string form (datetime limitation) + d = value if isinstance(value, util.Date) else util.Date(value) + return d.days_from_epoch + SimpleDateType.EPOCH_OFFSET_DAYS + + +class Time(Column): + """ + Stores a timezone-naive time-of-day, with nanosecond precision + + .. versionadded:: 2.6.0 + + requires C* 2.2+ and protocol v4+ + """ + db_type = 'time' def to_database(self, value): - value = super(Date, self).to_database(value) + value = super(Time, self).to_database(value) if value is None: return - if isinstance(value, datetime): - value = value.date() - if not isinstance(value, date): - raise ValidationError("{} '{}' is not a date object".format(self.column_name, repr(value))) - - return int((value - date(1970, 1, 1)).total_seconds() * 1000) + # str(util.Time) yields desired CQL encoding + return value if isinstance(value, util.Time) else util.Time(value) class UUID(Column): @@ -852,7 +867,7 @@ class UserDefinedType(Column): def __init__(self, user_type, **kwargs): """ - :param type user_type: specifies the :class:`~.UserType` model of the column + :param type user_type: specifies the :class:`~.cqlengine.usertype.UserType` model of the column """ self.user_type = user_type self.db_type = "frozen<%s>" % user_type.type_name() diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 6e9306c7cf..db0011344d 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -638,27 +638,29 @@ class SimpleDateType(_CassandraType): typename = 'date' date_format = "%Y-%m-%d" + # Values of the 'date'` type are encoded as 32-bit unsigned integers + # representing a number of days with epoch (January 1st, 1970) at the center of the + # range (2^31). + EPOCH_OFFSET_DAYS = 2 ** 31 + @classmethod def validate(cls, val): if not isinstance(val, util.Date): val = util.Date(val) return val + @staticmethod + def deserialize(byts, protocol_version): + days = uint32_unpack(byts) - SimpleDateType.EPOCH_OFFSET_DAYS + return util.Date(days) + @staticmethod def serialize(val, protocol_version): - # Values of the 'date'` type are encoded as 32-bit unsigned integers - # representing a number of days with epoch (January 1st, 1970) at the center of the - # range (2^31). try: days = val.days_from_epoch except AttributeError: days = util.Date(val).days_from_epoch - return uint32_pack(days + 2 ** 31) - - @staticmethod - def deserialize(byts, protocol_version): - days = uint32_unpack(byts) - 2 ** 31 - return util.Date(days) + return uint32_pack(days + SimpleDateType.EPOCH_OFFSET_DAYS) class ShortType(_CassandraType): @@ -682,6 +684,10 @@ def validate(cls, val): val = util.Time(val) return val + @staticmethod + def deserialize(byts, protocol_version): + return util.Time(int64_unpack(byts)) + @staticmethod def serialize(val, protocol_version): try: @@ -690,10 +696,6 @@ def serialize(val, protocol_version): nano = util.Time(val).nanosecond_time return int64_pack(nano) - @staticmethod - def deserialize(byts, protocol_version): - return util.Time(int64_unpack(byts)) - class UTF8Type(_CassandraType): typename = 'text' diff --git a/cassandra/util.py b/cassandra/util.py index 35e1e5e40f..dd4633596d 100644 --- a/cassandra/util.py +++ b/cassandra/util.py @@ -917,7 +917,7 @@ def __str__(self): class Date(object): ''' - Idealized naive date: year, month, day + Idealized date: year, month, day Offers wider year range than datetime.date. For Dates that cannot be represented as a datetime.date (because datetime.MINYEAR, datetime.MAXYEAR), this type falls back @@ -997,5 +997,5 @@ def __str__(self): dt = datetime_from_timestamp(self.seconds) return "%04d-%02d-%02d" % (dt.year, dt.month, dt.day) except: - # If we overflow datetime.[MIN|M + # If we overflow datetime.[MIN|MAX] return str(self.days_from_epoch) diff --git a/docs/api/cassandra/cqlengine/columns.rst b/docs/api/cassandra/cqlengine/columns.rst index a214d2cd94..aad164b606 100644 --- a/docs/api/cassandra/cqlengine/columns.rst +++ b/docs/api/cassandra/cqlengine/columns.rst @@ -52,7 +52,7 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: Counter -.. autoclass:: Date(**kwargs) +.. autoclass:: Date .. autoclass:: DateTime(**kwargs) @@ -70,10 +70,16 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: Set +.. autoclass:: SmallInt + .. autoclass:: Text +.. autoclass:: Time + .. autoclass:: TimeUUID(**kwargs) +.. autoclass:: TinyInt + .. autoclass:: UserDefinedType .. autoclass:: UUID(**kwargs) From f23cc3a46a3372b2f6984c5e96a6852c4fa60964 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 26 May 2015 10:57:24 -0500 Subject: [PATCH 0148/2431] Add Date type to CQL encoder --- cassandra/encoder.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cassandra/encoder.py b/cassandra/encoder.py index d9c3e85212..2d2433333e 100644 --- a/cassandra/encoder.py +++ b/cassandra/encoder.py @@ -29,7 +29,7 @@ import six from cassandra.util import (OrderedDict, OrderedMap, OrderedMapSerializedKey, - sortedset, Time) + sortedset, Time, Date) if six.PY3: long = int @@ -76,6 +76,7 @@ def __init__(self): datetime.datetime: self.cql_encode_datetime, datetime.date: self.cql_encode_date, datetime.time: self.cql_encode_time, + Date: self.cql_encode_date_ext, Time: self.cql_encode_time, dict: self.cql_encode_map_collection, OrderedDict: self.cql_encode_map_collection, @@ -163,11 +164,18 @@ def cql_encode_date(self, val): def cql_encode_time(self, val): """ - Converts a :class:`datetime.date` object to a string with format + Converts a :class:`cassandra.util.Time` object to a string with format ``HH:MM:SS.mmmuuunnn``. """ return "'%s'" % val + def cql_encode_date_ext(self, val): + """ + Encodes a :class:`cassandra.util.Date` object as an integer + """ + # using the int form in case the Date exceeds datetime.[MIN|MAX]YEAR + return str(val.days_from_epoch + 2 ** 31) + def cql_encode_sequence(self, val): """ Converts a sequence to a string of the form ``(item1, item2, ...)``. This From d3626fb8cd50358969250d934662e40b45c6b2a7 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 26 May 2015 10:58:05 -0500 Subject: [PATCH 0149/2431] cqle: tweak existing Date tests to expect Date type PYTHON-245 --- .../cqlengine/columns/test_validation.py | 21 +++++++++---------- .../cqlengine/columns/test_value_io.py | 12 +++++------ .../cqlengine/model/test_model_io.py | 9 ++++---- .../integration/cqlengine/model/test_udts.py | 3 ++- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index 8d4959b80d..94d0d156e4 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -18,10 +18,6 @@ from uuid import uuid4, uuid1 from cassandra import InvalidRequest -from cassandra.cqlengine.management import sync_table, drop_table -from cassandra.cqlengine.models import Model, ValidationError -from cassandra.cqlengine.connection import execute - from cassandra.cqlengine.columns import TimeUUID from cassandra.cqlengine.columns import Text from cassandra.cqlengine.columns import Integer @@ -33,6 +29,10 @@ from cassandra.cqlengine.columns import Boolean from cassandra.cqlengine.columns import Decimal from cassandra.cqlengine.columns import Inet +from cassandra.cqlengine.connection import execute +from cassandra.cqlengine.management import sync_table, drop_table +from cassandra.cqlengine.models import Model, ValidationError +from cassandra import util from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -55,7 +55,7 @@ def tearDownClass(cls): def test_datetime_io(self): now = datetime.now() - dt = self.DatetimeTest.objects.create(test_id=0, created_at=now) + self.DatetimeTest.objects.create(test_id=0, created_at=now) dt2 = self.DatetimeTest.objects(test_id=0).first() assert dt2.created_at.timetuple()[:6] == now.timetuple()[:6] @@ -164,16 +164,15 @@ def tearDownClass(cls): def test_date_io(self): today = date.today() self.DateTest.objects.create(test_id=0, created_at=today) - dt2 = self.DateTest.objects(test_id=0).first() - assert dt2.created_at.isoformat() == today.isoformat() + result = self.DateTest.objects(test_id=0).first() + self.assertEqual(result.created_at, util.Date(today)) def test_date_io_using_datetime(self): now = datetime.utcnow() self.DateTest.objects.create(test_id=0, created_at=now) - dt2 = self.DateTest.objects(test_id=0).first() - assert not isinstance(dt2.created_at, datetime) - assert isinstance(dt2.created_at, date) - assert dt2.created_at.isoformat() == now.date().isoformat() + result = self.DateTest.objects(test_id=0).first() + self.assertIsInstance(result.created_at, util.Date) + self.assertEqual(result.created_at, util.Date(now)) def test_date_none(self): self.DateTest.objects.create(test_id=1, created_at=None) diff --git a/tests/integration/cqlengine/columns/test_value_io.py b/tests/integration/cqlengine/columns/test_value_io.py index 17f96e2242..9b5947dff3 100644 --- a/tests/integration/cqlengine/columns/test_value_io.py +++ b/tests/integration/cqlengine/columns/test_value_io.py @@ -17,13 +17,13 @@ from uuid import uuid1, uuid4, UUID import six -from tests.integration.cqlengine.base import BaseCassEngTestCase - +from cassandra.cqlengine import columns from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.management import drop_table from cassandra.cqlengine.models import Model -from cassandra.cqlengine import columns -import unittest +from cassandra import util + +from tests.integration.cqlengine.base import BaseCassEngTestCase class BaseColumnIOTest(BaseCassEngTestCase): @@ -151,9 +151,9 @@ class TestDate(BaseColumnIOTest): column = columns.Date - now = datetime.now().date() + now = util.Date(datetime.now().date()) pkey_val = now - data_val = now + timedelta(days=1) + data_val = util.Date(now.days_from_epoch + 1) class TestUUID(BaseColumnIOTest): diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 1af9a8af0f..afee65d698 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -17,14 +17,15 @@ from datetime import date, datetime from decimal import Decimal from operator import itemgetter -from cassandra.cqlengine import CQLEngineException -from tests.integration.cqlengine.base import BaseCassEngTestCase +from cassandra.cqlengine import columns +from cassandra.cqlengine import CQLEngineException from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.management import drop_table from cassandra.cqlengine.models import Model -from cassandra.cqlengine import columns +from cassandra.util import Date +from tests.integration.cqlengine.base import BaseCassEngTestCase class TestModel(Model): @@ -161,7 +162,7 @@ class AllDatatypesModel(Model): sync_table(AllDatatypesModel) - input = ['ascii', 2 ** 63 - 1, bytearray(b'hello world'), True, date(1970, 1, 1), + input = ['ascii', 2 ** 63 - 1, bytearray(b'hello world'), True, Date(date(1970, 1, 1)), datetime.utcfromtimestamp(872835240), Decimal('12.3E+7'), 2.39, 3.4028234663852886e+38, '123.123.123.123', 2147483647, 'text', UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 3299fda310..af2abaa0e5 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -21,6 +21,7 @@ from cassandra.cqlengine.usertype import UserType from cassandra.cqlengine import columns from cassandra.cqlengine.management import sync_table, sync_type, create_keyspace_simple, drop_keyspace +from cassandra.util import Date from tests.integration import get_server_versions from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -273,7 +274,7 @@ class AllDatatypesModel(Model): sync_table(AllDatatypesModel) - input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1), + input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=Date(date(1970, 1, 1)), f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l='text', m=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), n=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), From 0d2a8b4727eef3f863b8273a1f37c111b5898133 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 26 May 2015 12:10:28 -0500 Subject: [PATCH 0150/2431] Test infinite max attempts for ConstantReconnectionPolicy PYTHON-325 --- tests/unit/test_policies.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_policies.py b/tests/unit/test_policies.py index 3e24f71fac..4f865fb1ee 100644 --- a/tests/unit/test_policies.py +++ b/tests/unit/test_policies.py @@ -799,6 +799,14 @@ def test_schedule_negative_max_attempts(self): except ValueError: pass + def test_schedule_infinite_attempts(self): + delay = 2 + max_attempts = None + crp = ConstantReconnectionPolicy(delay=delay, max_attempts=max_attempts) + # this is infinite. we'll just verify one more than default + for _, d in zip(range(65), crp.new_schedule()): + self.assertEqual(d, delay) + class ExponentialReconnectionPolicyTest(unittest.TestCase): From 3cd9d50b1637f48ca8c10657e38fded3dd72f76b Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Tue, 26 May 2015 10:32:46 -0700 Subject: [PATCH 0151/2431] [PYTHON-160] Test for default loadbalancing policy --- .../long/test_loadbalancingpolicies.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index cf5f3a191b..7b2b87957d 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -17,6 +17,7 @@ from cassandra import ConsistencyLevel, Unavailable, OperationTimedOut, ReadTimeout from cassandra.cluster import Cluster, NoHostAvailable from cassandra.concurrent import execute_concurrent_with_args +from cassandra.metadata import murmur3 from cassandra.policies import (RoundRobinPolicy, DCAwareRoundRobinPolicy, TokenAwarePolicy, WhiteListRoundRobinPolicy) from cassandra.query import SimpleStatement @@ -84,6 +85,30 @@ def _query(self, session, keyspace, count=12, log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb + def test_token_aware_is_used_by_default(self): + """ + Test for default loadbalacing policy + + test_token_aware_is_used_by_default tests that the default loadbalancing policy is policies.TokenAwarePolicy. + It creates a simple Cluster and verifies that the default loadbalancing policy is TokenAwarePolicy if the + murmur3 C extension is found. Otherwise, the default loadbalancing policy is DCAwareRoundRobinPolicy. + + @since 2.6.0 + @jira_ticket PYTHON-160 + @expected_result TokenAwarePolicy should be the default loadbalancing policy. + + @test_category load_balancing:token_aware + """ + + cluster = Cluster(protocol_version=PROTOCOL_VERSION) + + if murmur3 is not None: + self.assertTrue(isinstance(cluster.load_balancing_policy, TokenAwarePolicy)) + else: + self.assertTrue(isinstance(cluster.load_balancing_policy, DCAwareRoundRobinPolicy)) + + cluster.shutdown() + def test_roundrobin(self): use_singledc() keyspace = 'test_roundrobin' From 330349ae572a7c8f06f27b992e1d53c27293c6fe Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 26 May 2015 16:16:19 -0500 Subject: [PATCH 0152/2431] Update type tests --- cassandra/util.py | 12 ++ tests/integration/datatype_utils.py | 31 +++--- tests/integration/standard/test_types.py | 135 ++++++++++------------- 3 files changed, 82 insertions(+), 96 deletions(-) diff --git a/cassandra/util.py b/cassandra/util.py index dd4633596d..16457df589 100644 --- a/cassandra/util.py +++ b/cassandra/util.py @@ -896,6 +896,9 @@ def _from_time(self, t): t.second * Time.SECOND + t.microsecond * Time.MICRO) + def __hash__(self): + return self.nanosecond_time + def __eq__(self, other): if isinstance(other, Time): return self.nanosecond_time == other.nanosecond_time @@ -907,6 +910,9 @@ def __eq__(self, other): datetime.time(hour=self.hour, minute=self.minute, second=self.second, microsecond=self.nanosecond // Time.MICRO) == other + def __lt__(self, other): + return self.nanosecond_time < other.nanosecond_time + def __repr__(self): return "Time(%s)" % self.nanosecond_time @@ -977,6 +983,9 @@ def _from_datestring(self, s): dt = datetime.datetime.strptime(s, self.date_format) self._from_timetuple(dt.timetuple()) + def __hash__(self): + return self.days_from_epoch + def __eq__(self, other): if isinstance(other, Date): return self.days_from_epoch == other.days_from_epoch @@ -989,6 +998,9 @@ def __eq__(self, other): except Exception: return False + def __lt__(self, other): + return self.days_from_epoch < other.days_from_epoch + def __repr__(self): return "Date(%s)" % self.days_from_epoch diff --git a/tests/integration/datatype_utils.py b/tests/integration/datatype_utils.py index a7d8aeb1f8..80087b67f1 100644 --- a/tests/integration/datatype_utils.py +++ b/tests/integration/datatype_utils.py @@ -16,17 +16,12 @@ from datetime import datetime, date, time from uuid import uuid1, uuid4 -try: - from blist import sortedset -except ImportError: - sortedset = set # noqa - -from cassandra.util import OrderedMap +from cassandra.util import OrderedMap, Date, Time, sortedset from tests.integration import get_server_versions -PRIMITIVE_DATATYPES = [ +PRIMITIVE_DATATYPES = sortedset([ 'ascii', 'bigint', 'blob', @@ -42,26 +37,26 @@ 'uuid', 'varchar', 'varint', -] +]) -COLLECTION_TYPES = [ +COLLECTION_TYPES = sortedset([ 'list', 'set', 'map', -] +]) def update_datatypes(): _cass_version, _cql_version = get_server_versions() if _cass_version >= (2, 1, 0): - COLLECTION_TYPES.append('tuple') + COLLECTION_TYPES.add('tuple') + + if _cass_version >= (2, 2, 0): + PRIMITIVE_DATATYPES.update(['date', 'time', 'smallint', 'tinyint']) - if _cass_version >= (3, 0, 0): - PRIMITIVE_DATATYPES.append('date') - PRIMITIVE_DATATYPES.append('time') - PRIMITIVE_DATATYPES.append('tinyint') - PRIMITIVE_DATATYPES.append('smallint') + global SAMPLE_DATA + SAMPLE_DATA = get_sample_data() def get_sample_data(): @@ -114,10 +109,10 @@ def get_sample_data(): sample_data[datatype] = int(str(2147483647) + '000') elif datatype == 'date': - sample_data[datatype] = date(2015, 1, 15) + sample_data[datatype] = Date(date(2015, 1, 15)) elif datatype == 'time': - sample_data[datatype] = time(16, 47, 25, 7) + sample_data[datatype] = Time(time(16, 47, 25, 7)) elif datatype == 'tinyint': sample_data[datatype] = 123 diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index c4821d9c87..ba2a9738a8 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -41,31 +41,28 @@ def setup_module(): class TypeTests(unittest.TestCase): - def setUp(self): - self._cass_version, self._cql_version = get_server_versions() - - self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) - self.session = self.cluster.connect() - self.session.execute("CREATE KEYSPACE typetests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") - self.cluster.shutdown() - - def tearDown(self): - self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) - self.session = self.cluster.connect() - self.session.execute("DROP KEYSPACE typetests") - self.cluster.shutdown() + @classmethod + def setUpClass(cls): + cls._cass_version, cls._cql_version = get_server_versions() + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.session = cls.cluster.connect() + cls.session.execute("CREATE KEYSPACE typetests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") + cls.session.set_keyspace("typetests") + + @classmethod + def tearDownClass(cls): + cls.session.execute("DROP KEYSPACE typetests") + cls.cluster.shutdown() def test_can_insert_blob_type_as_string(self): """ Tests that byte strings in Python maps to blob type in Cassandra """ - - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session s.execute("CREATE TABLE blobstring (a ascii PRIMARY KEY, b blob)") - params = ['key1', b'blobyblob'] + params = ['key1', b'blobbyblob'] query = "INSERT INTO blobstring (a, b) VALUES (%s, %s)" # In python2, with Cassandra > 2.0, we don't treat the 'byte str' type as a blob, so we'll encode it @@ -98,9 +95,7 @@ def test_can_insert_blob_type_as_bytearray(self): """ Tests that blob type in Cassandra maps to bytearray in Python """ - - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session s.execute("CREATE TABLE blobbytes (a ascii PRIMARY KEY, b blob)") @@ -111,13 +106,10 @@ def test_can_insert_blob_type_as_bytearray(self): for expected, actual in zip(params, results): self.assertEqual(expected, actual) - c.shutdown() - def test_can_insert_primitive_datatypes(self): """ Test insertion of all datatype primitives """ - c = Cluster(protocol_version=PROTOCOL_VERSION) s = c.connect("typetests") @@ -254,54 +246,62 @@ def test_can_insert_empty_strings_and_nulls(self): """ Test insertion of empty strings and null values """ - - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session # create table alpha_type_list = ["zz int PRIMARY KEY"] col_names = [] + string_types = set(('ascii', 'text', 'varchar')) + string_columns = set(('')) + # this is just a list of types to try with empty strings + non_string_types = PRIMITIVE_DATATYPES - string_types - set(('blob', 'date', 'inet', 'time', 'timestamp')) + non_string_columns = set() start_index = ord('a') for i, datatype in enumerate(PRIMITIVE_DATATYPES): - alpha_type_list.append("{0} {1}".format(chr(start_index + i), datatype)) - col_names.append(chr(start_index + i)) + col_name = chr(start_index + i) + alpha_type_list.append("{0} {1}".format(col_name, datatype)) + col_names.append(col_name) + if datatype in non_string_types: + non_string_columns.add(col_name) + if datatype in string_types: + string_columns.add(col_name) - s.execute("CREATE TABLE alltypes ({0})".format(', '.join(alpha_type_list))) + s.execute("CREATE TABLE all_empty ({0})".format(', '.join(alpha_type_list))) # verify all types initially null with simple statement columns_string = ','.join(col_names) - s.execute("INSERT INTO alltypes (zz) VALUES (2)") - results = s.execute("SELECT {0} FROM alltypes WHERE zz=2".format(columns_string))[0] + s.execute("INSERT INTO all_empty (zz) VALUES (2)") + results = s.execute("SELECT {0} FROM all_empty WHERE zz=2".format(columns_string))[0] self.assertTrue(all(x is None for x in results)) # verify all types initially null with prepared statement - select = s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)) + select = s.prepare("SELECT {0} FROM all_empty WHERE zz=?".format(columns_string)) results = s.execute(select.bind([2]))[0] self.assertTrue(all(x is None for x in results)) # insert empty strings for string-like fields - expected_values = {'j': '', 'a': '', 'n': ''} - columns_string = ','.join(expected_values.keys()) - placeholders = ','.join(["%s"] * len(expected_values)) - s.execute("INSERT INTO alltypes (zz, {0}) VALUES (3, {1})".format(columns_string, placeholders), expected_values.values()) + expected_values = dict((col, '') for col in string_columns) + columns_string = ','.join(string_columns) + placeholders = ','.join(["%s"] * len(string_columns)) + s.execute("INSERT INTO all_empty (zz, {0}) VALUES (3, {1})".format(columns_string, placeholders), expected_values.values()) # verify string types empty with simple statement - results = s.execute("SELECT {0} FROM alltypes WHERE zz=3".format(columns_string))[0] + results = s.execute("SELECT {0} FROM all_empty WHERE zz=3".format(columns_string))[0] for expected, actual in zip(expected_values.values(), results): self.assertEqual(actual, expected) # verify string types empty with prepared statement - results = s.execute(s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)), [3])[0] + results = s.execute(s.prepare("SELECT {0} FROM all_empty WHERE zz=?".format(columns_string)), [3])[0] for expected, actual in zip(expected_values.values(), results): self.assertEqual(actual, expected) # non-string types shouldn't accept empty strings - for col in ('b', 'd', 'e', 'f', 'g', 'i', 'l', 'm', 'o'): - query = "INSERT INTO alltypes (zz, {0}) VALUES (4, %s)".format(col) + for col in non_string_columns: + query = "INSERT INTO all_empty (zz, {0}) VALUES (4, %s)".format(col) with self.assertRaises(InvalidRequest): s.execute(query, ['']) - insert = s.prepare("INSERT INTO alltypes (zz, {0}) VALUES (4, ?)".format(col)) + insert = s.prepare("INSERT INTO all_empty (zz, {0}) VALUES (4, ?)".format(col)) with self.assertRaises(TypeError): s.execute(insert, ['']) @@ -314,7 +314,7 @@ def test_can_insert_empty_strings_and_nulls(self): # insert the data columns_string = ','.join(col_names) placeholders = ','.join(["%s"] * len(col_names)) - simple_insert = "INSERT INTO alltypes (zz, {0}) VALUES (5, {1})".format(columns_string, placeholders) + simple_insert = "INSERT INTO all_empty (zz, {0}) VALUES (5, {1})".format(columns_string, placeholders) s.execute(simple_insert, params) # then insert None, which should null them out @@ -322,13 +322,13 @@ def test_can_insert_empty_strings_and_nulls(self): s.execute(simple_insert, null_values) # check via simple statement - query = "SELECT {0} FROM alltypes WHERE zz=5".format(columns_string) + query = "SELECT {0} FROM all_empty WHERE zz=5".format(columns_string) results = s.execute(query)[0] for col in results: self.assertEqual(None, col) # check via prepared statement - select = s.prepare("SELECT {0} FROM alltypes WHERE zz=?".format(columns_string)) + select = s.prepare("SELECT {0} FROM all_empty WHERE zz=?".format(columns_string)) results = s.execute(select.bind([5]))[0] for col in results: self.assertEqual(None, col) @@ -337,7 +337,7 @@ def test_can_insert_empty_strings_and_nulls(self): s.execute(simple_insert, params) placeholders = ','.join(["?"] * len(col_names)) - insert = s.prepare("INSERT INTO alltypes (zz, {0}) VALUES (5, {1})".format(columns_string, placeholders)) + insert = s.prepare("INSERT INTO all_empty (zz, {0}) VALUES (5, {1})".format(columns_string, placeholders)) s.execute(insert, null_values) results = s.execute(query)[0] @@ -348,15 +348,11 @@ def test_can_insert_empty_strings_and_nulls(self): for col in results: self.assertEqual(None, col) - s.shutdown() - def test_can_insert_empty_values_for_int32(self): """ Ensure Int32Type supports empty values """ - - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session s.execute("CREATE TABLE empty_values (a text PRIMARY KEY, b int)") s.execute("INSERT INTO empty_values (a, b) VALUES ('a', blobAsInt(0x))") @@ -367,8 +363,6 @@ def test_can_insert_empty_values_for_int32(self): finally: Int32Type.support_empty_values = False - c.shutdown() - def test_timezone_aware_datetimes_are_timestamps(self): """ Ensure timezone-aware datetimes are converted to timestamps correctly @@ -383,8 +377,7 @@ def test_timezone_aware_datetimes_are_timestamps(self): eastern_tz = pytz.timezone('US/Eastern') eastern_tz.localize(dt) - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session s.execute("CREATE TABLE tz_aware (a ascii PRIMARY KEY, b timestamp)") @@ -399,8 +392,6 @@ def test_timezone_aware_datetimes_are_timestamps(self): result = s.execute("SELECT b FROM tz_aware WHERE a='key2'")[0].b self.assertEqual(dt.utctimetuple(), result.utctimetuple()) - c.shutdown() - def test_can_insert_tuples(self): """ Basic test of tuple functionality @@ -508,17 +499,16 @@ def test_can_insert_tuples_all_primitive_datatypes(self): "k int PRIMARY KEY, " "v frozen>)" % ','.join(PRIMITIVE_DATATYPES)) - for i in range(len(PRIMITIVE_DATATYPES)): + values = [] + type_count = len(PRIMITIVE_DATATYPES) + for i, data_type in enumerate(PRIMITIVE_DATATYPES): # create tuples to be written and ensure they match with the expected response # responses have trailing None values for every element that has not been written - created_tuple = [get_sample(PRIMITIVE_DATATYPES[j]) for j in range(i + 1)] - response_tuple = tuple(created_tuple + [None for j in range(len(PRIMITIVE_DATATYPES) - i - 1)]) - written_tuple = tuple(created_tuple) - - s.execute("INSERT INTO tuple_primitive (k, v) VALUES (%s, %s)", (i, written_tuple)) - + values.append(get_sample(data_type)) + expected = tuple(values + [None] * (type_count - len(values))) + s.execute("INSERT INTO tuple_primitive (k, v) VALUES (%s, %s)", (i, tuple(values))) result = s.execute("SELECT v FROM tuple_primitive WHERE k=%s", (i,))[0] - self.assertEqual(response_tuple, result.v) + self.assertEqual(result.v, expected) c.shutdown() def test_can_insert_tuples_all_collection_datatypes(self): @@ -668,8 +658,7 @@ def test_can_insert_tuples_with_nulls(self): if self._cass_version < (2, 1, 0): raise unittest.SkipTest("The tuple type was introduced in Cassandra 2.1") - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session s.execute("CREATE TABLE tuples_nulls (k int PRIMARY KEY, t frozen>)") @@ -688,28 +677,20 @@ def test_can_insert_tuples_with_nulls(self): self.assertEqual(('', None, None, b''), result[0].t) self.assertEqual(('', None, None, b''), s.execute(read)[0].t) - c.shutdown() - def test_can_insert_unicode_query_string(self): """ Test to ensure unicode strings can be used in a query """ - - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session query = u"SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = 'ef\u2052ef' AND columnfamily_name = %s" s.execute(query, (u"fe\u2051fe",)) - c.shutdown() - def test_can_read_composite_type(self): """ Test to ensure that CompositeTypes can be used in a query """ - - c = Cluster(protocol_version=PROTOCOL_VERSION) - s = c.connect("typetests") + s = self.session s.execute(""" CREATE TABLE composites ( @@ -728,5 +709,3 @@ def test_can_read_composite_type(self): result = s.execute("SELECT * FROM composites WHERE a = 0")[0] self.assertEqual(0, result.a) self.assertEqual(('abc',), result.b) - - c.shutdown() From abdbf454190f86baeaec72cfe0753db779ac3dc2 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 26 May 2015 16:28:01 -0500 Subject: [PATCH 0153/2431] UNSET Prepared test updates peer review input --- .../standard/test_prepared_statements.py | 49 ++++++++++--------- tests/unit/test_parameter_binding.py | 2 - 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index 8c50d7b747..42c3894e83 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -28,6 +28,7 @@ def setup_module(): use_singledc() + class PreparedStatementTests(unittest.TestCase): def test_basic(self): @@ -65,9 +66,9 @@ def test_basic(self): session.execute(bound) prepared = session.prepare( - """ - SELECT * FROM cf0 WHERE a=? - """) + """ + SELECT * FROM cf0 WHERE a=? + """) self.assertIsInstance(prepared, PreparedStatement) bound = prepared.bind(('a')) @@ -90,9 +91,9 @@ def test_basic(self): session.execute(bound) prepared = session.prepare( - """ - SELECT * FROM cf0 WHERE a=? - """) + """ + SELECT * FROM cf0 WHERE a=? + """) self.assertIsInstance(prepared, PreparedStatement) @@ -216,9 +217,9 @@ def test_none_values(self): session.execute(bound) prepared = session.prepare( - """ - SELECT * FROM test3rf.test WHERE k=? - """) + """ + SELECT * FROM test3rf.test WHERE k=? + """) self.assertIsInstance(prepared, PreparedStatement) bound = prepared.bind((1,)) @@ -249,8 +250,8 @@ def test_unset_values(self): # table with at least two values so one can be used as a marker session.execute("CREATE TABLE IF NOT EXISTS test1rf.test_unset_values (k int PRIMARY KEY, v0 int, v1 int)") - insert = session.prepare( "INSERT INTO test1rf.test_unset_values (k, v0, v1) VALUES (?, ?, ?)") - select = session.prepare( "SELECT * FROM test1rf.test_unset_values WHERE k=?") + insert = session.prepare("INSERT INTO test1rf.test_unset_values (k, v0, v1) VALUES (?, ?, ?)") + select = session.prepare("SELECT * FROM test1rf.test_unset_values WHERE k=?") bind_expected = [ # initial condition @@ -273,6 +274,8 @@ def test_unset_values(self): results = session.execute(select, (0,)) self.assertEqual(results[0], expected) + self.assertRaises(ValueError, session.execute, select, (UNSET_VALUE, 0, 0)) + cluster.shutdown() def test_no_meta(self): @@ -289,9 +292,9 @@ def test_no_meta(self): session.execute(bound) prepared = session.prepare( - """ - SELECT * FROM test3rf.test WHERE k=0 - """) + """ + SELECT * FROM test3rf.test WHERE k=0 + """) self.assertIsInstance(prepared, PreparedStatement) bound = prepared.bind(None) @@ -319,9 +322,9 @@ def test_none_values_dicts(self): session.execute(bound) prepared = session.prepare( - """ - SELECT * FROM test3rf.test WHERE k=? - """) + """ + SELECT * FROM test3rf.test WHERE k=? + """) self.assertIsInstance(prepared, PreparedStatement) bound = prepared.bind({'k': 1}) @@ -348,9 +351,9 @@ def test_async_binding(self): future.result() prepared = session.prepare( - """ - SELECT * FROM test3rf.test WHERE k=? - """) + """ + SELECT * FROM test3rf.test WHERE k=? + """) self.assertIsInstance(prepared, PreparedStatement) future = session.execute_async(prepared, (873,)) @@ -377,9 +380,9 @@ def test_async_binding_dicts(self): future.result() prepared = session.prepare( - """ - SELECT * FROM test3rf.test WHERE k=? - """) + """ + SELECT * FROM test3rf.test WHERE k=? + """) self.assertIsInstance(prepared, PreparedStatement) future = session.execute_async(prepared, {'k': 873}) diff --git a/tests/unit/test_parameter_binding.py b/tests/unit/test_parameter_binding.py index 6b84260279..a53a9f6dfb 100644 --- a/tests/unit/test_parameter_binding.py +++ b/tests/unit/test_parameter_binding.py @@ -146,8 +146,6 @@ def test_dict_missing_routing_key(self): def test_missing_value(self): self.assertRaises(KeyError, self.bound.bind, {'rk0': 0, 'rk1': 0, 'ck0': 0}) - # legacy behavior: short parameters are allowed through, errored by server - #self.assertRaises(ValueError, self.bound.bind, (0, 0, 0)) def test_dict_extra_value(self): self.assertRaises(ValueError, self.bound.bind, {'rk0': 0, 'rk1': 0, 'ck0': 0, 'v0': 0, 'should_not_be_here': 123}) From 05b60071fb713b5229015ec7c640207f2156bbf9 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Tue, 26 May 2015 13:59:29 -0500 Subject: [PATCH 0154/2431] [PYTHON-311] Test for inserting collections of UDTs --- .../integration/cqlengine/model/test_udts.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index af2abaa0e5..0d94d5d096 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -15,7 +15,7 @@ from datetime import date, datetime from decimal import Decimal import unittest -from uuid import UUID +from uuid import UUID, uuid4 from cassandra.cqlengine.models import Model from cassandra.cqlengine.usertype import UserType @@ -286,3 +286,43 @@ class AllDatatypesModel(Model): for i in range(ord('a'), ord('a') + 15): self.assertEqual(input[chr(i)], output[chr(i)]) + + def test_nested_udts_inserts(self): + """ + Test for inserting collections of user types using cql engine. + + test_nested_udts_inserts Constructs a model that contains a list of usertypes. It will then attempt to insert + them. The expectation is that no exception is thrown during insert. For sanity sake we also validate that our + input and output values match. This combination of model, and UT produces a syntax error in 2.5.1 due to + improper quoting around the names collection. + + @since 2.6.0 + @jira_ticket PYTHON-311 + @expected_result No syntax exception thrown + + @test_category data_types:udt + """ + + class Name(UserType): + type_name__ = "header" + + name = columns.Text() + value = columns.Text() + + class Container(Model): + id = columns.UUID(primary_key=True, default=uuid4) + names = columns.List(columns.UserDefinedType(Name())) + + # Construct the objects and insert them + names = [] + for i in range(0, 10): + names.append(Name(name="name{0}".format(i), value="value{0}".format(i))) + + # Create table, insert data + sync_table(Container) + Container.create(id=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), names=names) + + # Validate input and output matches + self.assertEqual(1, Container.objects.count()) + names_output = Container.objects().first().names + self.assertEqual(names_output, names) From dca25dac784bb6657422e959392b4ba5765d522e Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Tue, 26 May 2015 15:49:22 -0700 Subject: [PATCH 0155/2431] [PYTHON-245] Tests for Date, Time, SmallInt, TinyInt in cqlengine --- .../cqlengine/model/test_model_io.py | 26 ++++++++------ .../integration/cqlengine/model/test_udts.py | 36 +++++++++++-------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index afee65d698..1c274353ac 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -14,7 +14,7 @@ from uuid import uuid4, UUID import random -from datetime import date, datetime +from datetime import datetime, date, time from decimal import Decimal from operator import itemgetter @@ -23,7 +23,7 @@ from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.management import drop_table from cassandra.cqlengine.models import Model -from cassandra.util import Date +from cassandra.util import Date, Time from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -155,24 +155,28 @@ class AllDatatypesModel(Model): i = columns.Float(double_precision=False) j = columns.Inet() k = columns.Integer() - l = columns.Text() - m = columns.TimeUUID() - n = columns.UUID() - o = columns.VarInt() + l = columns.SmallInt() + m = columns.Text() + n = columns.Time() + o = columns.TimeUUID() + p = columns.TinyInt() + q = columns.UUID() + r = columns.VarInt() sync_table(AllDatatypesModel) input = ['ascii', 2 ** 63 - 1, bytearray(b'hello world'), True, Date(date(1970, 1, 1)), datetime.utcfromtimestamp(872835240), Decimal('12.3E+7'), 2.39, - 3.4028234663852886e+38, '123.123.123.123', 2147483647, 'text', - UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + 3.4028234663852886e+38, '123.123.123.123', 2147483647, 32523, 'text', Time(time(16, 47, 25, 7)), + UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), 123, UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), int(str(2147483647) + '000')] AllDatatypesModel.create(id=0, a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1), f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, - i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l='text', - m=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), n=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), - o=int(str(2147483647) + '000')) + i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l=32523, m='text', + n=time(16, 47, 25, 7), o=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), + p=123, q=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + r=int(str(2147483647) + '000')) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first() diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index af2abaa0e5..84220cd036 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import date, datetime +from datetime import datetime, date, time from decimal import Decimal import unittest from uuid import UUID @@ -21,7 +21,7 @@ from cassandra.cqlengine.usertype import UserType from cassandra.cqlengine import columns from cassandra.cqlengine.management import sync_table, sync_type, create_keyspace_simple, drop_keyspace -from cassandra.util import Date +from cassandra.util import Date, Time from tests.integration import get_server_versions from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -216,10 +216,13 @@ class AllDatatypes(UserType): i = columns.Float(double_precision=False) j = columns.Inet() k = columns.Integer() - l = columns.Text() - m = columns.TimeUUID() - n = columns.UUID() - o = columns.VarInt() + l = columns.SmallInt() + m = columns.Text() + n = columns.Time() + o = columns.TimeUUID() + p = columns.TinyInt() + q = columns.UUID() + r = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) @@ -227,7 +230,8 @@ class AllDatatypesModel(Model): sync_table(AllDatatypesModel) - input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, l=None, m=None, n=None) + input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, + l=None, m=None, n=None, o=None, p=None, q=None, r=None) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) @@ -263,10 +267,13 @@ class AllDatatypes(UserType): i = columns.Float(double_precision=False) j = columns.Inet() k = columns.Integer() - l = columns.Text() - m = columns.TimeUUID() - n = columns.UUID() - o = columns.VarInt() + l = columns.SmallInt() + m = columns.Text() + n = columns.Time() + o = columns.TimeUUID() + p = columns.TinyInt() + q = columns.UUID() + r = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) @@ -276,9 +283,10 @@ class AllDatatypesModel(Model): input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=Date(date(1970, 1, 1)), f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, - i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l='text', - m=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), n=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), - o=int(str(2147483647) + '000')) + i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l=32523, m='text', + n=Time(time(16, 47, 25, 7)), o=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), + p=123, q=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + r=int(str(2147483647) + '000')) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) From c0b8c1351aefc254893ed69e92906bf319c43c59 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Tue, 26 May 2015 20:35:40 -0700 Subject: [PATCH 0156/2431] Fix failing cqlengine tests related to protocol v4 datatypes --- .../cqlengine/columns/test_validation.py | 6 +- .../cqlengine/columns/test_value_io.py | 63 +++++++++- .../cqlengine/model/test_model_io.py | 87 ++++++++++---- .../integration/cqlengine/model/test_udts.py | 112 +++++++++++------- 4 files changed, 197 insertions(+), 71 deletions(-) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index 94d0d156e4..0d790bfc60 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta, date, tzinfo from decimal import Decimal as D -from unittest import TestCase +from unittest import TestCase, SkipTest from uuid import uuid4, uuid1 from cassandra import InvalidRequest @@ -34,6 +34,7 @@ from cassandra.cqlengine.models import Model, ValidationError from cassandra import util +from tests.integration import PROTOCOL_VERSION from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -153,6 +154,9 @@ class DateTest(Model): @classmethod def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + super(TestDate, cls).setUpClass() sync_table(cls.DateTest) diff --git a/tests/integration/cqlengine/columns/test_value_io.py b/tests/integration/cqlengine/columns/test_value_io.py index 9b5947dff3..243f7096ad 100644 --- a/tests/integration/cqlengine/columns/test_value_io.py +++ b/tests/integration/cqlengine/columns/test_value_io.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime, timedelta +from datetime import datetime, timedelta, time from decimal import Decimal from uuid import uuid1, uuid4, UUID import six +import unittest from cassandra.cqlengine import columns from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.management import drop_table from cassandra.cqlengine.models import Model -from cassandra import util +from cassandra.util import Date, Time + +from tests.integration import PROTOCOL_VERSION from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -55,10 +58,10 @@ def setUpClass(cls): # create a table with the given column class IOTestModel(Model): - table_name = cls.column.db_type + "_io_test_model_{}".format(uuid4().hex[:8]) pkey = cls.column(primary_key=True) data = cls.column() + cls._generated_model = IOTestModel sync_table(cls._generated_model) @@ -149,11 +152,18 @@ class TestDateTime(BaseColumnIOTest): class TestDate(BaseColumnIOTest): + @classmethod + def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + super(TestDate, cls).setUpClass() + column = columns.Date - now = util.Date(datetime.now().date()) + now = Date(datetime.now().date()) pkey_val = now - data_val = util.Date(now.days_from_epoch + 1) + data_val = Date(now.days_from_epoch + 1) class TestUUID(BaseColumnIOTest): @@ -210,3 +220,46 @@ class TestDecimalIO(BaseColumnIOTest): def comparator_converter(self, val): return Decimal(val) +class TestTime(BaseColumnIOTest): + + @classmethod + def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + super(TestTime, cls).setUpClass() + + column = columns.Time + + pkey_val = Time(time(2, 12, 7, 48)) + data_val = Time(time(16, 47, 25, 7)) + + +class TestSmallInt(BaseColumnIOTest): + + @classmethod + def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + super(TestSmallInt, cls).setUpClass() + + column = columns.SmallInt + + pkey_val = 16768 + data_val = 32523 + + +class TestTinyInt(BaseColumnIOTest): + + @classmethod + def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + super(TestTinyInt, cls).setUpClass() + + column = columns.TinyInt + + pkey_val = 1 + data_val = 123 \ No newline at end of file diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 1c274353ac..436e8955fb 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -14,6 +14,7 @@ from uuid import uuid4, UUID import random +import unittest from datetime import datetime, date, time from decimal import Decimal from operator import itemgetter @@ -25,6 +26,7 @@ from cassandra.cqlengine.models import Model from cassandra.util import Date, Time +from tests.integration import PROTOCOL_VERSION from tests.integration.cqlengine.base import BaseCassEngTestCase class TestModel(Model): @@ -148,40 +150,72 @@ class AllDatatypesModel(Model): b = columns.BigInt() c = columns.Blob() d = columns.Boolean() - e = columns.Date() - f = columns.DateTime() - g = columns.Decimal() - h = columns.Double() - i = columns.Float(double_precision=False) - j = columns.Inet() - k = columns.Integer() - l = columns.SmallInt() - m = columns.Text() - n = columns.Time() - o = columns.TimeUUID() - p = columns.TinyInt() - q = columns.UUID() - r = columns.VarInt() + e = columns.DateTime() + f = columns.Decimal() + g = columns.Double() + h = columns.Float(double_precision=False) + i = columns.Inet() + j = columns.Integer() + k = columns.Text() + l = columns.TimeUUID() + m = columns.UUID() + n = columns.VarInt() sync_table(AllDatatypesModel) - input = ['ascii', 2 ** 63 - 1, bytearray(b'hello world'), True, Date(date(1970, 1, 1)), - datetime.utcfromtimestamp(872835240), Decimal('12.3E+7'), 2.39, - 3.4028234663852886e+38, '123.123.123.123', 2147483647, 32523, 'text', Time(time(16, 47, 25, 7)), - UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), 123, UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), + input = ['ascii', 2 ** 63 - 1, bytearray(b'hello world'), True, datetime.utcfromtimestamp(872835240), + Decimal('12.3E+7'), 2.39, 3.4028234663852886e+38, '123.123.123.123', 2147483647, 'text', + UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), int(str(2147483647) + '000')] - AllDatatypesModel.create(id=0, a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1), - f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, - i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l=32523, m='text', - n=time(16, 47, 25, 7), o=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), - p=123, q=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), - r=int(str(2147483647) + '000')) + AllDatatypesModel.create(id=0, a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, + e=datetime.utcfromtimestamp(872835240), f=Decimal('12.3E+7'), g=2.39, + h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647, k='text', + l=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), + m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000')) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first() - for i, i_char in enumerate(range(ord('a'), ord('a') + 15)): + for i, i_char in enumerate(range(ord('a'), ord('a') + 14)): + self.assertEqual(input[i], output[chr(i_char)]) + + def test_can_insert_model_with_all_protocol_v4_column_types(self): + """ + Test for inserting all protocol v4 column types into a Model + + test_can_insert_model_with_all_protocol_v4_column_types tests that each cqlengine protocol v4 column type can be + inserted into a Model. It first creates a Model that has each cqlengine protocol v4 column type. It then creates + a Model instance where all the fields have corresponding data, which performs the insert into the Cassandra table. + Finally, it verifies that each column read from the Model from Cassandra is the same as the input parameters. + + @since 2.6.0 + @jira_ticket PYTHON-245 + @expected_result The Model is inserted with each protocol v4 column type, and the resulting read yields proper data for each column. + + @test_category data_types:primitive + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + class AllDatatypesModel(Model): + id = columns.Integer(primary_key=True) + a = columns.Date() + b = columns.SmallInt() + c = columns.Time() + d = columns.TinyInt() + + sync_table(AllDatatypesModel) + + input = [Date(date(1970, 1, 1)), 32523, Time(time(16, 47, 25, 7)), 123] + + AllDatatypesModel.create(id=0, a=date(1970, 1, 1), b=32523, c=time(16, 47, 25, 7), d=123) + + self.assertEqual(1, AllDatatypesModel.objects.count()) + output = AllDatatypesModel.objects().first() + + for i, i_char in enumerate(range(ord('a'), ord('a') + 3)): self.assertEqual(input[i], output[chr(i_char)]) def test_can_insert_double_and_float(self): @@ -399,6 +433,9 @@ class TestQuerying(BaseCassEngTestCase): @classmethod def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Date query tests require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + super(TestQuerying, cls).setUpClass() drop_table(TestQueryModel) sync_table(TestQueryModel) diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 681f7ba8f1..49e934c53e 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -23,7 +23,7 @@ from cassandra.cqlengine.management import sync_table, sync_type, create_keyspace_simple, drop_keyspace from cassandra.util import Date, Time -from tests.integration import get_server_versions +from tests.integration import get_server_versions, PROTOCOL_VERSION from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -31,8 +31,8 @@ class UserDefinedTypeTests(BaseCassEngTestCase): @classmethod def setUpClass(self): - if get_server_versions()[0] < (2, 1, 0): - raise unittest.SkipTest("UDTs require Cassandra 2.1 or greater") + if PROTOCOL_VERSION < 3: + raise unittest.SkipTest("UDTs require native protocol 3+, currently using: {0}".format(PROTOCOL_VERSION)) def test_can_create_udts(self): class User(UserType): @@ -209,20 +209,16 @@ class AllDatatypes(UserType): b = columns.BigInt() c = columns.Blob() d = columns.Boolean() - e = columns.Date() - f = columns.DateTime() - g = columns.Decimal() - h = columns.Double() - i = columns.Float(double_precision=False) - j = columns.Inet() - k = columns.Integer() - l = columns.SmallInt() - m = columns.Text() - n = columns.Time() - o = columns.TimeUUID() - p = columns.TinyInt() - q = columns.UUID() - r = columns.VarInt() + e = columns.DateTime() + f = columns.Decimal() + g = columns.Double() + h = columns.Float(double_precision=False) + i = columns.Inet() + j = columns.Integer() + k = columns.Text() + l = columns.TimeUUID() + m = columns.UUID() + n = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) @@ -231,7 +227,7 @@ class AllDatatypesModel(Model): sync_table(AllDatatypesModel) input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, - l=None, m=None, n=None, o=None, p=None, q=None, r=None) + l=None, m=None, n=None) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) @@ -260,20 +256,16 @@ class AllDatatypes(UserType): b = columns.BigInt() c = columns.Blob() d = columns.Boolean() - e = columns.Date() - f = columns.DateTime() - g = columns.Decimal() - h = columns.Double() - i = columns.Float(double_precision=False) - j = columns.Inet() - k = columns.Integer() - l = columns.SmallInt() - m = columns.Text() - n = columns.Time() - o = columns.TimeUUID() - p = columns.TinyInt() - q = columns.UUID() - r = columns.VarInt() + e = columns.DateTime() + f = columns.Decimal() + g = columns.Double() + h = columns.Float(double_precision=False) + i = columns.Inet() + j = columns.Integer() + k = columns.Text() + l = columns.TimeUUID() + m = columns.UUID() + n = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) @@ -281,18 +273,58 @@ class AllDatatypesModel(Model): sync_table(AllDatatypesModel) - input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=Date(date(1970, 1, 1)), - f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=2.39, - i=3.4028234663852886e+38, j='123.123.123.123', k=2147483647, l=32523, m='text', - n=Time(time(16, 47, 25, 7)), o=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), - p=123, q=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), - r=int(str(2147483647) + '000')) + input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, + e=datetime.utcfromtimestamp(872835240), f=Decimal('12.3E+7'), g=2.39, + h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647, k='text', + l=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), + m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000')) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first().data - for i in range(ord('a'), ord('a') + 15): + for i in range(ord('a'), ord('a') + 14): + self.assertEqual(input[chr(i)], output[chr(i)]) + + def test_can_insert_udts_protocol_v4_datatypes(self): + """ + Test for inserting all protocol v4 column types into a UserType + + test_can_insert_udts_protocol_v4_datatypes tests that each protocol v4 cqlengine column type can be inserted + into a UserType. It first creates a UserType that has each protocol v4 cqlengine column type, and a corresponding + table/Model. It then creates a UserType instance where all the fields have corresponding data, and inserts the + UserType as an instance of the Model. Finally, it verifies that each column read from the UserType from Cassandra + is the same as the input parameters. + + @since 2.6.0 + @jira_ticket PYTHON-245 + @expected_result The UserType is inserted with each protocol v4 column type, and the resulting read yields proper data for each column. + + @test_category data_types:udt + """ + + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes in UDTs require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + class AllDatatypes(UserType): + a = columns.Date() + b = columns.SmallInt() + c = columns.Time() + d = columns.TinyInt() + + class AllDatatypesModel(Model): + id = columns.Integer(primary_key=True) + data = columns.UserDefinedType(AllDatatypes) + + sync_table(AllDatatypesModel) + + input = AllDatatypes(a=Date(date(1970, 1, 1)), b=32523, c=Time(time(16, 47, 25, 7)), d=123) + AllDatatypesModel.create(id=0, data=input) + + self.assertEqual(1, AllDatatypesModel.objects.count()) + output = AllDatatypesModel.objects().first().data + + for i in range(ord('a'), ord('a') + 3): self.assertEqual(input[chr(i)], output[chr(i)]) def test_nested_udts_inserts(self): @@ -319,7 +351,7 @@ class Name(UserType): class Container(Model): id = columns.UUID(primary_key=True, default=uuid4) - names = columns.List(columns.UserDefinedType(Name())) + names = columns.List(columns.UserDefinedType(Name)) # Construct the objects and insert them names = [] From f00e5ea350a41d5aa72d21a7c25d218259244bd0 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 22 May 2015 15:26:20 -0500 Subject: [PATCH 0157/2431] Run with Mirroring query handler if C* >= 2.2 moved custom payload tests to .../standard --- tests/integration/__init__.py | 7 ++- .../{long => standard}/test_custom_payload.py | 43 +++---------------- 2 files changed, 11 insertions(+), 39 deletions(-) rename tests/integration/{long => standard}/test_custom_payload.py (81%) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 887498e398..1f43d6d652 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -203,9 +203,14 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True): common.switch_cluster(path, cluster_name) cluster.populate(nodes, ipformat=ipformat) + jvm_args = [] + # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back + if PROTOCOL_VERSION >= 4: + jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"] + if start: log.debug("Starting ccm %s cluster", cluster_name) - cluster.start(wait_for_binary_proto=True, wait_other_notice=True) + cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args) setup_keyspace(ipformat=ipformat) CCM_CLUSTER = cluster diff --git a/tests/integration/long/test_custom_payload.py b/tests/integration/standard/test_custom_payload.py similarity index 81% rename from tests/integration/long/test_custom_payload.py rename to tests/integration/standard/test_custom_payload.py index f131a5e756..c253d64b4b 100644 --- a/tests/integration/long/test_custom_payload.py +++ b/tests/integration/standard/test_custom_payload.py @@ -21,54 +21,21 @@ from cassandra.query import (SimpleStatement, BatchStatement, BatchType) from cassandra.cluster import Cluster -from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace - +from tests.integration import use_singledc, PROTOCOL_VERSION def setup_module(): - """ - We need some custom setup for this module. All unit tests in this module - require protocol >=4. We won't bother going through the setup required unless that is the - protocol version we are using. - """ - - # If we aren't at protocol v 4 or greater don't waste time setting anything up, all tests will be skipped - if PROTOCOL_VERSION >= 4: - # Don't start the ccm cluster until we get the custom jvm argument specified - use_singledc(start=False) - ccm_cluster = get_cluster() - # if needed stop CCM cluster - ccm_cluster.stop() - # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back to us - jmv_args = [ - " -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"] - ccm_cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jmv_args) - # wait for nodes to startup - setup_keyspace() - - -def teardown_module(): - """ - The rests of the tests don't need our custom payload query handle so stop the cluster so we - don't impact other tests - """ - - ccm_cluster = get_cluster() - if ccm_cluster is not None: - ccm_cluster.stop() - + use_singledc() class CustomPayloadTests(unittest.TestCase): - def setUp(self): - """ - Test is skipped if run with cql version <4 - """ - + @classmethod + def setUpClass(cls): if PROTOCOL_VERSION < 4: raise unittest.SkipTest( "Native protocol 4,0+ is required for custom payloads, currently using %r" % (PROTOCOL_VERSION,)) + def setUp(self): self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) self.session = self.cluster.connect() From 0eb6476aafa5f219c1718210374850227d5f923f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 26 May 2015 09:03:26 -0500 Subject: [PATCH 0158/2431] Integration test for client warning PYTHON-315 --- .../standard/test_client_warnings.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 tests/integration/standard/test_client_warnings.py diff --git a/tests/integration/standard/test_client_warnings.py b/tests/integration/standard/test_client_warnings.py new file mode 100644 index 0000000000..b329887239 --- /dev/null +++ b/tests/integration/standard/test_client_warnings.py @@ -0,0 +1,124 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import unittest2 as unittest +except ImportError: + import unittest + +from cassandra.query import BatchStatement +from cassandra.cluster import Cluster + +from tests.integration import use_singledc, PROTOCOL_VERSION + + +def setup_module(): + use_singledc() + + +class ClientWarningTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest( + "Native protocol 4,0+ is required for client warnings, currently using %r" + % (PROTOCOL_VERSION,)) + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.session = cls.cluster.connect() + + cls.session.execute("CREATE TABLE IF NOT EXISTS test1rf.client_warning (k int, v0 int, v1 int, PRIMARY KEY (k, v0))") + cls.prepared = cls.session.prepare("INSERT INTO test1rf.client_warning (k, v0, v1) VALUES (?, ?, ?)") + + cls.warn_batch = BatchStatement() + # 213 = 5 * 1024 / (4+4 + 4+4 + 4+4) + # thresh_kb/ (min param size) + for x in range(213): + cls.warn_batch.add(cls.prepared, (x, x, 1)) + + + @classmethod + def tearDownClass(cls): + cls.cluster.shutdown() + + def test_warning_basic(self): + """ + Test to validate that client warnings can be surfaced + + @since 2.6.0 + @jira_ticket PYTHON-315 + @expected_result valid warnings returned + @test_assumptions + - batch_size_warn_threshold_in_kb: 5 + @test_category queries:client_warning + """ + future = self.session.execute_async(self.warn_batch) + future.result() + self.assertEqual(len(future.warnings), 1) + self.assertRegexpMatches(future.warnings[0], 'Batch.*exceeding.*') + + def test_warning_with_trace(self): + """ + Test to validate client warning with tracing + + @since 2.6.0 + @jira_ticket PYTHON-315 + @expected_result valid warnings returned + @test_assumptions + - batch_size_warn_threshold_in_kb: 5 + @test_category queries:client_warning + """ + future = self.session.execute_async(self.warn_batch, trace=True) + future.result() + self.assertEqual(len(future.warnings), 1) + self.assertRegexpMatches(future.warnings[0], 'Batch.*exceeding.*') + self.assertIsNotNone(future._query_trace) + + def test_warning_with_custom_payload(self): + """ + Test to validate client warning with custom payload + + @since 2.6.0 + @jira_ticket PYTHON-315 + @expected_result valid warnings returned + @test_assumptions + - batch_size_warn_threshold_in_kb: 5 + @test_category queries:client_warning + """ + payload = {'key': 'value'} + future = self.session.execute_async(self.warn_batch, custom_payload=payload) + future.result() + self.assertEqual(len(future.warnings), 1) + self.assertRegexpMatches(future.warnings[0], 'Batch.*exceeding.*') + self.assertDictEqual(future.custom_payload, payload) + + def test_warning_with_trace_and_custom_payload(self): + """ + Test to validate client warning with tracing and client warning + + @since 2.6.0 + @jira_ticket PYTHON-315 + @expected_result valid warnings returned + @test_assumptions + - batch_size_warn_threshold_in_kb: 5 + @test_category queries:client_warning + """ + payload = {'key': 'value'} + future = self.session.execute_async(self.warn_batch, trace=True, custom_payload=payload) + future.result() + self.assertEqual(len(future.warnings), 1) + self.assertRegexpMatches(future.warnings[0], 'Batch.*exceeding.*') + self.assertIsNotNone(future._query_trace) + self.assertDictEqual(future.custom_payload, payload) From 9b149b30925355da94bf1eb1cab17196c2dc4fcd Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 27 May 2015 13:35:28 -0500 Subject: [PATCH 0159/2431] Don't skipTest in unittest.TestCase.setUpClass resolves an issue where skipping from class setup errors in unittest2 This is reported fixed in unittest2 0.4.2, but seems to be still happening in 1.0.1 https://pypi.python.org/pypi/unittest2#id9 --- tests/integration/standard/test_client_warnings.py | 14 +++++++++++--- tests/integration/standard/test_custom_payload.py | 5 +---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/integration/standard/test_client_warnings.py b/tests/integration/standard/test_client_warnings.py index b329887239..173cc69756 100644 --- a/tests/integration/standard/test_client_warnings.py +++ b/tests/integration/standard/test_client_warnings.py @@ -33,9 +33,8 @@ class ClientWarningTests(unittest.TestCase): @classmethod def setUpClass(cls): if PROTOCOL_VERSION < 4: - raise unittest.SkipTest( - "Native protocol 4,0+ is required for client warnings, currently using %r" - % (PROTOCOL_VERSION,)) + return + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) cls.session = cls.cluster.connect() @@ -51,8 +50,17 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + if PROTOCOL_VERSION < 4: + return + cls.cluster.shutdown() + def setUp(self): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest( + "Native protocol 4,0+ is required for client warnings, currently using %r" + % (PROTOCOL_VERSION,)) + def test_warning_basic(self): """ Test to validate that client warnings can be surfaced diff --git a/tests/integration/standard/test_custom_payload.py b/tests/integration/standard/test_custom_payload.py index c253d64b4b..87b38fac40 100644 --- a/tests/integration/standard/test_custom_payload.py +++ b/tests/integration/standard/test_custom_payload.py @@ -28,14 +28,11 @@ def setup_module(): class CustomPayloadTests(unittest.TestCase): - @classmethod - def setUpClass(cls): + def setUp(self): if PROTOCOL_VERSION < 4: raise unittest.SkipTest( "Native protocol 4,0+ is required for custom payloads, currently using %r" % (PROTOCOL_VERSION,)) - - def setUp(self): self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) self.session = self.cluster.connect() From f04a14d5728492c1cc7df72210bbe26b6a597755 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Wed, 27 May 2015 14:41:56 -0500 Subject: [PATCH 0160/2431] Adding documentation for metadata tests --- tests/integration/standard/test_metadata.py | 152 ++++++++++++++++++-- 1 file changed, 143 insertions(+), 9 deletions(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 69d599f534..2bb88b50cf 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -1357,20 +1357,31 @@ def make_function_kwargs(self, called_on_null=True): 'called_on_null_input': called_on_null} def test_functions_after_udt(self): + """ + Test to to ensure udt's come after functions in in keyspace dump + + test_functions_after_udt creates a basic function. Then queries that function and make sure that in the results + that UDT's are listed before any corresponding functions, when we dump the keyspace + Ideally we would make a function that takes a udt type, but this presently fails because C* c059a56 requires + udt to be frozen to create, but does not store meta indicating frozen + SEE https://issues.apache.org/jira/browse/CASSANDRA-9186 + Maybe update this after release + kwargs = self.make_function_kwargs() + kwargs['type_signature'][0] = "frozen<%s>" % udt_name + expected_meta = Function(**kwargs) + with self.VerifiedFunction(self, **kwargs): + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result UDT's should come before any functions + @test_category function + """ + self.assertNotIn(self.function_name, self.keyspace_function_meta) udt_name = 'udtx' self.session.execute("CREATE TYPE %s (x int)" % udt_name) - # Ideally we would make a function that takes a udt type, but - # this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen - # https://issues.apache.org/jira/browse/CASSANDRA-9186 - # Maybe update this after release - #kwargs = self.make_function_kwargs() - #kwargs['type_signature'][0] = "frozen<%s>" % udt_name - - #expected_meta = Function(**kwargs) - #with self.VerifiedFunction(self, **kwargs): with self.VerifiedFunction(self, **self.make_function_kwargs()): # udts must come before functions in keyspace dump keyspace_cql = self.cluster.metadata.keyspaces[self.keyspace_name].export_as_string() @@ -1380,22 +1391,54 @@ def test_functions_after_udt(self): self.assertGreater(func_idx, type_idx) def test_function_same_name_diff_types(self): + """ + Test to verify to that functions with different signatures are differentiated in metadata + + test_function_same_name_diff_types Creates two functions. One with the same name but a slightly different + signature. Then ensures that both are surfaced separately in our metadata. + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result function with the same name but different signatures should be surfaced separately + @test_category function + """ + + # Create a function kwargs = self.make_function_kwargs() with self.VerifiedFunction(self, **kwargs): + # another function: same name, different type sig. self.assertGreater(len(kwargs['type_signature']), 1) self.assertGreater(len(kwargs['argument_names']), 1) kwargs['type_signature'] = kwargs['type_signature'][:1] kwargs['argument_names'] = kwargs['argument_names'][:1] + + # Ensure they are surfaced separately with self.VerifiedFunction(self, **kwargs): functions = [f for f in self.keyspace_function_meta.values() if f.name == self.function_name] self.assertEqual(len(functions), 2) self.assertNotEqual(functions[0].type_signature, functions[1].type_signature) def test_functions_follow_keyspace_alter(self): + """ + Test to verify to that functions maintain equality after a keyspace is altered + + test_functions_follow_keyspace_alter creates a function then alters a the keyspace associated with that function. + After the alter we validate that the function maintains the same metadata + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result functions are the same after parent keyspace is altered + @test_category function + """ + + # Create function with self.VerifiedFunction(self, **self.make_function_kwargs()): original_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] self.session.execute('ALTER KEYSPACE %s WITH durable_writes = false' % self.keyspace_name) + + # After keyspace alter ensure that we maintain function equality. try: new_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) @@ -1404,6 +1447,20 @@ def test_functions_follow_keyspace_alter(self): self.session.execute('ALTER KEYSPACE %s WITH durable_writes = true' % self.keyspace_name) def test_function_cql_called_on_null(self): + """ + Test to verify to that that called on null argument is honored on function creation. + + test_functions_follow_keyspace_alter create two functions. One with the called_on_null_input set to true, + the other with it set to false. We then verify that the metadata constructed from those function is correctly + reflected + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result functions metadata correctly reflects called_on_null_input flag. + @test_category function + """ + kwargs = self.make_function_kwargs() kwargs['called_on_null_input'] = True with self.VerifiedFunction(self, **kwargs) as vf: @@ -1460,10 +1517,36 @@ def make_aggregate_kwargs(self, state_func, state_type, final_func=None, init_co 'return_type': "does not matter for creation"} def test_return_type_meta(self): + """ + Test to verify to that the return type of a an aggregate is honored in the metadata + + test_return_type_meta creates an aggregate then ensures the return type of the created + aggregate is correctly surfaced in the metadata + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result aggregate has the correct return typ in the metadata + @test_category aggregate + """ + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=1)) as va: self.assertIs(self.keyspace_aggregate_meta[va.signature].return_type, Int32Type) def test_init_cond(self): + """ + Test to verify that various initial conditions are correctly surfaced in various aggregate functions + + test_init_cond creates several different types of aggregates, and given various initial conditions it verifies that + they correctly impact the aggregate's execution + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result initial conditions are correctly evaluated as part of the aggregates + @test_category aggregate + """ + # This is required until the java driver bundled with C* is updated to support v4 c = Cluster(protocol_version=3) s = c.connect(self.keyspace_name) @@ -1496,6 +1579,19 @@ def test_init_cond(self): c.shutdown() def test_aggregates_after_functions(self): + """ + Test to verify that aggregates are listed after function in metadata + + test_aggregates_after_functions creates an aggregate, and then verifies that they are listed + after any function creations when the keypspace dump is preformed + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result aggregates are declared after any functions + @test_category aggregate + """ + # functions must come before functions in keyspace dump with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('extend_list', ListType.apply_parameters([UTF8Type]))): keyspace_cql = self.cluster.metadata.keyspaces[self.keyspace_name].export_as_string() @@ -1505,6 +1601,18 @@ def test_aggregates_after_functions(self): self.assertGreater(aggregate_idx, func_idx) def test_same_name_diff_types(self): + """ + Test to verify to that aggregates with different signatures are differentiated in metadata + + test_same_name_diff_types Creates two Aggregates. One with the same name but a slightly different + signature. Then ensures that both are surfaced separately in our metadata. + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result aggregates with the same name but different signatures should be surfaced separately + @test_category function + """ + kwargs = self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=0) with self.VerifiedAggregate(self, **kwargs): kwargs['state_func'] = 'sum_int_two' @@ -1515,6 +1623,19 @@ def test_same_name_diff_types(self): self.assertNotEqual(aggregates[0].type_signature, aggregates[1].type_signature) def test_aggregates_follow_keyspace_alter(self): + """ + Test to verify to that aggregates maintain equality after a keyspace is altered + + test_aggregates_follow_keyspace_alter creates a function then alters a the keyspace associated with that + function. After the alter we validate that the function maintains the same metadata + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result aggregates are the same after parent keyspace is altered + @test_category function + """ + with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('sum_int', Int32Type, init_cond=0)): original_keyspace_meta = self.cluster.metadata.keyspaces[self.keyspace_name] self.session.execute('ALTER KEYSPACE %s WITH durable_writes = false' % self.keyspace_name) @@ -1526,6 +1647,19 @@ def test_aggregates_follow_keyspace_alter(self): self.session.execute('ALTER KEYSPACE %s WITH durable_writes = true' % self.keyspace_name) def test_cql_optional_params(self): + """ + Test to verify that the initial_cond and final_func parameters are correctly honored + + test_cql_optional_params creates various aggregates with different combinations of initial_condition, + and final_func parameters set. It then ensures they are correctly honored. + + + @since 2.6.0 + @jira_ticket PYTHON-211 + @expected_result initial_condition and final_func parameters are honored correctly + @test_category function + """ + kwargs = self.make_aggregate_kwargs('extend_list', ListType.apply_parameters([UTF8Type])) # no initial condition, final func From c843e6886f3b2b17645117a8d8f1c8187078a254 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 27 May 2015 15:48:46 -0500 Subject: [PATCH 0161/2431] Add windows inet_pton for python versions that do not have the socket routine Fixes an issue with inet addresses in the protocol (PYTHON-309) Adds IPv6 support for Windows (PYTHON-20) --- cassandra/cqltypes.py | 22 ++++-------- cassandra/protocol.py | 5 +-- cassandra/util.py | 83 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index db0011344d..77fc2b911b 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -517,35 +517,25 @@ def serialize(byts, protocol_version): return varint_pack(byts) -have_ipv6_packing = hasattr(socket, 'inet_ntop') - - class InetAddressType(_CassandraType): typename = 'inet' - # TODO: implement basic ipv6 support for Windows? - # inet_ntop and inet_pton aren't available on Windows - @staticmethod def deserialize(byts, protocol_version): if len(byts) == 16: - if not have_ipv6_packing: - raise Exception( - "IPv6 addresses cannot currently be handled on Windows") - return socket.inet_ntop(socket.AF_INET6, byts) + return util.inet_ntop(socket.AF_INET6, byts) else: + # util.inet_pton could also handle, but this is faster + # since we've already determined the AF return socket.inet_ntoa(byts) @staticmethod def serialize(addr, protocol_version): if ':' in addr: - fam = socket.AF_INET6 - if not have_ipv6_packing: - raise Exception( - "IPv6 addresses cannot currently be handled on Windows") - return socket.inet_pton(fam, addr) + return util.inet_pton(socket.AF_INET6, addr) else: - fam = socket.AF_INET + # util.inet_pton could also handle, but this is faster + # since we've already determined the AF return socket.inet_aton(addr) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 13078b46fd..1c851f8c5d 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -39,6 +39,7 @@ TupleType, lookup_casstype, SimpleDateType, TimeType, ByteType, ShortType) from cassandra.policies import WriteType +from cassandra import util log = logging.getLogger(__name__) @@ -1133,7 +1134,7 @@ def read_inet(f): addrfam = socket.AF_INET6 else: raise InternalError("bad inet address: %r" % (addrbytes,)) - return (socket.inet_ntop(addrfam, addrbytes), port) + return (util.inet_ntop(addrfam, addrbytes), port) def write_inet(f, addrtuple): @@ -1142,7 +1143,7 @@ def write_inet(f, addrtuple): addrfam = socket.AF_INET6 else: addrfam = socket.AF_INET - addrbytes = socket.inet_pton(addrfam, addr) + addrbytes = util.inet_pton(addrfam, addr) write_byte(f, len(addrbytes)) f.write(addrbytes) write_int(f, port) diff --git a/cassandra/util.py b/cassandra/util.py index 16457df589..f188cdcb2f 100644 --- a/cassandra/util.py +++ b/cassandra/util.py @@ -1011,3 +1011,86 @@ def __str__(self): except: # If we overflow datetime.[MIN|MAX] return str(self.days_from_epoch) + +import socket +if hasattr(socket, 'inet_pton'): + inet_pton = socket.inet_pton + inet_ntop = socket.inet_ntop +else: + """ + Windows doesn't have socket.inet_pton and socket.inet_ntop until Python 3.4 + This is an alternative impl using ctypes, based on this win_inet_pton project: + https://github.com/hickeroar/win_inet_pton + """ + import ctypes + + + class sockaddr(ctypes.Structure): + _fields_ = [("sa_family", ctypes.c_short), + ("__pad1", ctypes.c_ushort), + ("ipv4_addr", ctypes.c_byte * 4), + ("ipv6_addr", ctypes.c_byte * 16), + ("__pad2", ctypes.c_ulong)] + + + if hasattr(ctypes, 'windll'): + WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA + WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA + else: + def not_windows(*args): + raise Exception("IPv6 addresses cannot be handled on Windows. " + "Missing ctypes.windll") + WSAStringToAddressA = not_windows + WSAAddressToStringA = not_windows + + + def inet_pton(address_family, ip_string): + if address_family == socket.AF_INET: + return socket.inet_aton(ip_string) + + addr = sockaddr() + addr.sa_family = address_family + addr_size = ctypes.c_int(ctypes.sizeof(addr)) + + if WSAStringToAddressA( + ip_string, + address_family, + None, + ctypes.byref(addr), + ctypes.byref(addr_size) + ) != 0: + raise socket.error(ctypes.FormatError()) + + if address_family == socket.AF_INET6: + return ctypes.string_at(addr.ipv6_addr, 16) + + raise socket.error('unknown address family') + + + def inet_ntop(address_family, packed_ip): + if address_family == socket.AF_INET: + return socket.inet_ntoa(packed_ip) + + addr = sockaddr() + addr.sa_family = address_family + addr_size = ctypes.c_int(ctypes.sizeof(addr)) + ip_string = ctypes.create_string_buffer(128) + ip_string_size = ctypes.c_int(ctypes.sizeof(ip_string)) + + if address_family == socket.AF_INET6: + if len(packed_ip) != ctypes.sizeof(addr.ipv6_addr): + raise socket.error('packed IP wrong length for inet_ntoa') + ctypes.memmove(addr.ipv6_addr, packed_ip, 16) + else: + raise socket.error('unknown address family') + + if WSAAddressToStringA( + ctypes.byref(addr), + addr_size, + None, + ip_string, + ctypes.byref(ip_string_size) + ) != 0: + raise socket.error(ctypes.FormatError()) + + return ip_string[:ip_string_size.value - 1] From 2e4fbba38e235952630f5c9d1b7860e2aec53295 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Wed, 27 May 2015 16:03:35 -0700 Subject: [PATCH 0162/2431] Jenkins stabilization fixes --- .../columns/test_container_columns.py | 36 +++++++++++++++---- .../integration/cqlengine/model/test_udts.py | 16 ++++----- tests/integration/standard/test_types.py | 2 -- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/tests/integration/cqlengine/columns/test_container_columns.py b/tests/integration/cqlengine/columns/test_container_columns.py index a9c8b35965..85a4255813 100644 --- a/tests/integration/cqlengine/columns/test_container_columns.py +++ b/tests/integration/cqlengine/columns/test_container_columns.py @@ -13,9 +13,10 @@ # limitations under the License. from datetime import datetime, timedelta -import json +import json, six, sys, traceback, logging from uuid import uuid4 -import six + +from cassandra import WriteTimeout from cassandra.cqlengine.models import Model, ValidationError import cassandra.cqlengine.columns as columns @@ -23,10 +24,10 @@ from tests.integration.cqlengine import is_prepend_reversed from tests.integration.cqlengine.base import BaseCassEngTestCase - -class TestSetModel(Model): +log = logging.getLogger(__name__) +class TestSetModel(Model): partition = columns.UUID(primary_key=True, default=uuid4) int_set = columns.Set(columns.Integer, required=False) text_set = columns.Set(columns.Text, required=False) @@ -124,7 +125,14 @@ def test_element_count_validation(self): """ Tests that big collections are detected and raise an exception. """ - TestSetModel.create(text_set={str(uuid4()) for i in range(65535)}) + while True: + try: + TestSetModel.create(text_set={str(uuid4()) for i in range(65535)}) + break + except WriteTimeout: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb with self.assertRaises(ValidationError): TestSetModel.create(text_set={str(uuid4()) for i in range(65536)}) @@ -235,7 +243,14 @@ def test_element_count_validation(self): """ Tests that big collections are detected and raise an exception. """ - TestListModel.create(text_list=[str(uuid4()) for i in range(65535)]) + while True: + try: + TestListModel.create(text_list=[str(uuid4()) for i in range(65535)]) + break + except WriteTimeout: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb with self.assertRaises(ValidationError): TestListModel.create(text_list=[str(uuid4()) for i in range(65536)]) @@ -410,7 +425,14 @@ def test_element_count_validation(self): """ Tests that big collections are detected and raise an exception. """ - TestMapModel.create(text_map={str(uuid4()): i for i in range(65535)}) + while True: + try: + TestMapModel.create(text_map={str(uuid4()): i for i in range(65535)}) + break + except WriteTimeout: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb with self.assertRaises(ValidationError): TestMapModel.create(text_map={str(uuid4()): i for i in range(65536)}) diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 49e934c53e..2d870d38ed 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -306,23 +306,23 @@ def test_can_insert_udts_protocol_v4_datatypes(self): if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Protocol v4 datatypes in UDTs require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - class AllDatatypes(UserType): + class Allv4Datatypes(UserType): a = columns.Date() b = columns.SmallInt() c = columns.Time() d = columns.TinyInt() - class AllDatatypesModel(Model): + class Allv4DatatypesModel(Model): id = columns.Integer(primary_key=True) - data = columns.UserDefinedType(AllDatatypes) + data = columns.UserDefinedType(Allv4Datatypes) - sync_table(AllDatatypesModel) + sync_table(Allv4DatatypesModel) - input = AllDatatypes(a=Date(date(1970, 1, 1)), b=32523, c=Time(time(16, 47, 25, 7)), d=123) - AllDatatypesModel.create(id=0, data=input) + input = Allv4Datatypes(a=Date(date(1970, 1, 1)), b=32523, c=Time(time(16, 47, 25, 7)), d=123) + Allv4DatatypesModel.create(id=0, data=input) - self.assertEqual(1, AllDatatypesModel.objects.count()) - output = AllDatatypesModel.objects().first().data + self.assertEqual(1, Allv4DatatypesModel.objects.count()) + output = Allv4DatatypesModel.objects().first().data for i in range(ord('a'), ord('a') + 3): self.assertEqual(input[chr(i)], output[chr(i)]) diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index ba2a9738a8..c6c8114b57 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -89,8 +89,6 @@ def test_can_insert_blob_type_as_string(self): for expected, actual in zip(params, results): self.assertEqual(expected, actual) - c.shutdown() - def test_can_insert_blob_type_as_bytearray(self): """ Tests that blob type in Cassandra maps to bytearray in Python From 2216c3105327ba84a3be69c6f7e83272ed3481c9 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Wed, 27 May 2015 18:38:00 -0700 Subject: [PATCH 0163/2431] change overlapping table names in cqle test v4 test --- tests/integration/cqlengine/model/test_model_io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 436e8955fb..2033d9dc92 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -199,21 +199,21 @@ def test_can_insert_model_with_all_protocol_v4_column_types(self): if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - class AllDatatypesModel(Model): + class v4DatatypesModel(Model): id = columns.Integer(primary_key=True) a = columns.Date() b = columns.SmallInt() c = columns.Time() d = columns.TinyInt() - sync_table(AllDatatypesModel) + sync_table(v4DatatypesModel) input = [Date(date(1970, 1, 1)), 32523, Time(time(16, 47, 25, 7)), 123] - AllDatatypesModel.create(id=0, a=date(1970, 1, 1), b=32523, c=time(16, 47, 25, 7), d=123) + v4DatatypesModel.create(id=0, a=date(1970, 1, 1), b=32523, c=time(16, 47, 25, 7), d=123) - self.assertEqual(1, AllDatatypesModel.objects.count()) - output = AllDatatypesModel.objects().first() + self.assertEqual(1, v4DatatypesModel.objects.count()) + output = v4DatatypesModel.objects().first() for i, i_char in enumerate(range(ord('a'), ord('a') + 3)): self.assertEqual(input[i], output[chr(i_char)]) From 33c71799c8c7b29734be15297578f7f75d9ff9be Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 28 May 2015 11:06:37 -0500 Subject: [PATCH 0164/2431] Update CQL keywords, expose in API docs. PYTHON-319 PYTHON-324 --- cassandra/metadata.py | 49 +++++++++++++++++++++++++-------- docs/api/cassandra/metadata.rst | 9 ++++++ tests/unit/test_metadata.py | 4 +-- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 8ede52dc1e..843d96f38e 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -37,19 +37,44 @@ log = logging.getLogger(__name__) -_keywords = set(( - 'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with', - 'limit', 'using', 'use', 'count', 'set', - 'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create', - 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop', - 'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type', - 'compact', 'storage', 'order', 'by', 'asc', 'desc', 'clustering', - 'token', 'writetime', 'map', 'list', 'to' +cql_keywords = set(( + 'add', 'aggregate', 'all', 'allow', 'alter', 'and', 'apply', 'as', 'asc', 'ascii', 'authorize', 'batch', 'begin', + 'bigint', 'blob', 'boolean', 'by', 'called', 'clustering', 'columnfamily', 'compact', 'contains', 'count', + 'counter', 'create', 'custom', 'date', 'decimal', 'delete', 'desc', 'describe', 'distinct', 'double', 'drop', + 'entries', 'execute', 'exists', 'filtering', 'finalfunc', 'float', 'from', 'frozen', 'full', 'function', + 'functions', 'grant', 'if', 'in', 'index', 'inet', 'infinity', 'initcond', 'input', 'insert', 'int', 'into', 'json', + 'key', 'keys', 'keyspace', 'keyspaces', 'language', 'limit', 'list', 'login', 'map', 'modify', 'nan', 'nologin', + 'norecursive', 'nosuperuser', 'not', 'null', 'of', 'on', 'options', 'or', 'order', 'password', 'permission', + 'permissions', 'primary', 'rename', 'replace', 'returns', 'revoke', 'role', 'roles', 'schema', 'select', 'set', + 'sfunc', 'smallint', 'static', 'storage', 'stype', 'superuser', 'table', 'text', 'time', 'timestamp', 'timeuuid', + 'tinyint', 'to', 'token', 'trigger', 'truncate', 'ttl', 'tuple', 'type', 'unlogged', 'update', 'use', 'user', + 'users', 'using', 'uuid', 'values', 'varchar', 'varint', 'where', 'with', 'writetime' )) - -_unreserved_keywords = set(( - 'key', 'clustering', 'ttl', 'compact', 'storage', 'type', 'values' +""" +Set of keywords in CQL. + +Derived from .../cassandra/src/java/org/apache/cassandra/cql3/Cql.g +""" + +cql_keywords_unreserved = set(( + 'aggregate', 'all', 'as', 'ascii', 'bigint', 'blob', 'boolean', 'called', 'clustering', 'compact', 'contains', + 'count', 'counter', 'custom', 'date', 'decimal', 'distinct', 'double', 'exists', 'filtering', 'finalfunc', 'float', + 'frozen', 'function', 'functions', 'inet', 'initcond', 'input', 'int', 'json', 'key', 'keys', 'keyspaces', + 'language', 'list', 'login', 'map', 'nologin', 'nosuperuser', 'options', 'password', 'permission', 'permissions', + 'returns', 'role', 'roles', 'sfunc', 'smallint', 'static', 'storage', 'stype', 'superuser', 'text', 'time', + 'timestamp', 'timeuuid', 'tinyint', 'trigger', 'ttl', 'tuple', 'type', 'user', 'users', 'uuid', 'values', 'varchar', + 'varint', 'writetime' )) +""" +Set of unreserved keywords in CQL. + +Derived from .../cassandra/src/java/org/apache/cassandra/cql3/Cql.g +""" + +cql_keywords_reserved = cql_keywords - cql_keywords_unreserved +""" +Set of reserved keywords in CQL. +""" class Metadata(object): @@ -1383,7 +1408,7 @@ def protect_value(value): def is_valid_name(name): if name is None: return False - if name.lower() in _keywords - _unreserved_keywords: + if name.lower() in cql_keywords_reserved: return False return valid_cql3_word_re.match(name) is not None diff --git a/docs/api/cassandra/metadata.rst b/docs/api/cassandra/metadata.rst index a89fd03ddb..291f63d5be 100644 --- a/docs/api/cassandra/metadata.rst +++ b/docs/api/cassandra/metadata.rst @@ -3,6 +3,15 @@ .. module:: cassandra.metadata +.. autodata:: cql_keywords + :annotation: + +.. autodata:: cql_keywords_unreserved + :annotation: + +.. autodata:: cql_keywords_reserved + :annotation: + .. autoclass:: Metadata () :members: :exclude-members: rebuild_schema, rebuild_token_map, add_host, remove_host, get_host diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 5ce78f2e25..8bdd428c65 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -210,8 +210,8 @@ def test_is_valid_name(self): self.assertEqual(is_valid_name('test1'), True) self.assertEqual(is_valid_name('1test1'), False) - non_valid_keywords = cassandra.metadata._keywords - cassandra.metadata._unreserved_keywords - for keyword in non_valid_keywords: + invalid_keywords = cassandra.metadata.cql_keywords - cassandra.metadata.cql_keywords_unreserved + for keyword in invalid_keywords: self.assertEqual(is_valid_name(keyword), False) From 6b6d31cf5622f94eddae3eb31a0a0373b22e934d Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 28 May 2015 12:53:08 -0500 Subject: [PATCH 0165/2431] cqle: replace deprecated refresh_schema with new fn --- cassandra/cqlengine/management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/cqlengine/management.py b/cassandra/cqlengine/management.py index 5323c8f93d..5d5e641459 100644 --- a/cassandra/cqlengine/management.py +++ b/cassandra/cqlengine/management.py @@ -302,7 +302,7 @@ def _sync_type(ks_name, type_model, omit_subtypes=None): log.debug("sync_type creating new type %s", type_name_qualified) cql = get_create_type(type_model, ks_name) execute(cql) - cluster.refresh_schema(keyspace=ks_name, usertype=type_name) + cluster.refresh_user_type_metadata(ks_name, type_name) type_model.register_for_keyspace(ks_name) else: defined_fields = defined_types[type_name].field_names From 58676ac9a25c943ac1420a3fddd8dd7b46767e1f Mon Sep 17 00:00:00 2001 From: Kracekumar Ramaraju Date: Fri, 29 May 2015 12:11:40 +0530 Subject: [PATCH 0166/2431] Use inbuilt UUID for validation --- cassandra/cqlengine/columns.py | 12 ++++++------ .../integration/cqlengine/columns/test_validation.py | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index 9edf0d6588..57a4ffd392 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -15,7 +15,6 @@ from copy import deepcopy, copy from datetime import date, datetime import logging -import re import six import warnings @@ -518,8 +517,6 @@ class UUID(Column): """ db_type = 'uuid' - re_uuid = re.compile(r'[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}') - def validate(self, value): val = super(UUID, self).validate(value) if val is None: @@ -527,9 +524,12 @@ def validate(self, value): from uuid import UUID as _UUID if isinstance(val, _UUID): return val - if isinstance(val, six.string_types) and self.re_uuid.match(val): - return _UUID(val) - raise ValidationError("{} {} is not a valid uuid".format(self.column_name, value)) + if isinstance(val, six.string_types): + try: + return _UUID(val) + except ValueError: + raise ValidationError("{} {} is not a valid uuid".format( + self.column_name, value)) def to_python(self, value): return self.validate(value) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index 0d790bfc60..a74f83a0de 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -240,6 +240,13 @@ def test_uuid_str_no_dashes(self): t1 = self.UUIDTest.get(test_id=1) assert a_uuid == t1.a_uuid + def test_uuid_with_upcase(self): + a_uuid = uuid4() + val = str(a_uuid).upper() + t0 = self.UUIDTest.create(test_id=0, a_uuid=val) + t1 = self.UUIDTest.get(test_id=0) + assert a_uuid == t1.a_uuid + class TestTimeUUID(BaseCassEngTestCase): class TimeUUIDTest(Model): From 8e565d06ab94de13ac43e00a81d9741d9250f2e8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 29 May 2015 11:03:39 -0500 Subject: [PATCH 0167/2431] ValidationError for invalid UUID types Follow-on to #335. Fixes issue where non-string or UUID types would return None from UUID.validate. Also removed test that did not do what it purported (previously inserting None for key instead of value, and failing for Validation of UUID and not key). --- cassandra/cqlengine/columns.py | 8 ++++++-- .../cqlengine/columns/test_container_columns.py | 8 +------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index 57a4ffd392..d418b11947 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -528,8 +528,10 @@ def validate(self, value): try: return _UUID(val) except ValueError: - raise ValidationError("{} {} is not a valid uuid".format( - self.column_name, value)) + # fall-through to error + pass + raise ValidationError("{} {} is not a valid uuid".format( + self.column_name, value)) def to_python(self, value): return self.validate(value) @@ -824,6 +826,8 @@ def validate(self, value): return if not isinstance(val, dict): raise ValidationError('{} {} is not a dict object'.format(self.column_name, val)) + if None in val: + raise ValidationError("{} None is not allowed in a map".format(self.column_name)) return {self.key_col.validate(k): self.value_col.validate(v) for k, v in val.items()} def to_python(self, value): diff --git a/tests/integration/cqlengine/columns/test_container_columns.py b/tests/integration/cqlengine/columns/test_container_columns.py index 85a4255813..6ec301eb67 100644 --- a/tests/integration/cqlengine/columns/test_container_columns.py +++ b/tests/integration/cqlengine/columns/test_container_columns.py @@ -345,8 +345,6 @@ def test_blind_list_updates_from_none(self): assert m3.int_list == [] class TestMapModel(Model): - - partition = columns.UUID(primary_key=True, default=uuid4) int_map = columns.Map(columns.Integer, columns.UUID, required=False) text_map = columns.Map(columns.Text, columns.DateTime, required=False) @@ -371,11 +369,7 @@ def test_empty_default(self): def test_add_none_as_map_key(self): with self.assertRaises(ValidationError): - TestMapModel.create(int_map={None:1}) - - def test_add_none_as_map_value(self): - with self.assertRaises(ValidationError): - TestMapModel.create(int_map={None:1}) + TestMapModel.create(int_map={None: uuid4()}) def test_empty_retrieve(self): tmp = TestMapModel.create() From 101a4ab76e4b39d0759471f2e2d3f52dc4edd744 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 29 May 2015 12:59:02 -0500 Subject: [PATCH 0168/2431] Exclude the libevwrapper extention for Windows --- setup.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index df3aef2797..e85d332d37 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import print_function +import os import sys import warnings @@ -37,9 +38,6 @@ from distutils.cmd import Command -import os -import warnings - try: import subprocess has_subprocess = True @@ -247,6 +245,26 @@ def run_setup(extensions): sys.argv = [a for a in sys.argv if a != "--no-libev"] extensions.remove(libev_ext) +is_windows = os.name == 'nt' +if is_windows: + # libev is difficult to build, and uses select in Windows. + try: + extensions.remove(libev_ext) + except ValueError: + pass + build_extensions.error_message = """ +=============================================================================== +WARNING: could not compile %s. + +The C extensions are not required for the driver to run, but they add support +for token-aware routing with the Murmur3Partitioner. + +On Windows, make sure Visual Studio or an SDK is installed, and your environment +is configured to build for the appropriate architecture (matching your Python runtime). +This is often a matter of using vcvarsall.bat from your install directory, or running +from a command prompt in the Visual Studio Tools Start Menu. +=============================================================================== +""" platform_unsupported_msg = \ """ From f247df78764b10440b61bc8e9fbe2ef1ad6f42c6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 29 May 2015 13:30:41 -0500 Subject: [PATCH 0169/2431] cqle: add Float/Double overload deprecation to upgrade guide --- docs/api/cassandra/cqlengine/columns.rst | 2 +- docs/cqlengine/upgrade_guide.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/api/cassandra/cqlengine/columns.rst b/docs/api/cassandra/cqlengine/columns.rst index aad164b606..7e3957e0cb 100644 --- a/docs/api/cassandra/cqlengine/columns.rst +++ b/docs/api/cassandra/cqlengine/columns.rst @@ -58,7 +58,7 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: Decimal(**kwargs) -.. autoclass:: Double +.. autoclass:: Double(**kwargs) .. autoclass:: Float diff --git a/docs/cqlengine/upgrade_guide.rst b/docs/cqlengine/upgrade_guide.rst index 3488902790..831f94d7cf 100644 --- a/docs/cqlengine/upgrade_guide.rst +++ b/docs/cqlengine/upgrade_guide.rst @@ -79,6 +79,12 @@ This upgrade served as a good juncture to deprecate certain API features and inv to new ones. The first released version does not change functionality -- only introduces deprecation warnings. Future releases will remove these features in favor of the alternatives. +Float/Double Overload +--------------------- +Previously there was no ``Double`` column type. Doubles were modeled by specifying ``Float(double_precision=True)``. +This inititializer parameter is now deprecated. Applications should use :class:`~.columns.Double` for CQL ``double``, and :class:`~.columns.Float` +for CQL ``float``. + Schema Management ----------------- ``cassandra.cqlengine.management.create_keyspace`` is deprecated. Instead, use the new replication-strategy-specific From b945fa6fe508a020589f04a2b5a32c8257d852de Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 29 May 2015 13:45:30 -0500 Subject: [PATCH 0170/2431] Add link to platform/runtime survey --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index e976f93a66..bdc6dca7eb 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,8 @@ The driver supports Python 2.6, 2.7, 3.3, and 3.4*. * cqlengine component presently supports Python 2.7+ +**Help us focus our efforts!** Provide your input on the `Platform and Runtime Survey `_. + Installation ------------ Installation through pip is recommended:: From f00bfe746d3b8c8d97c7332e5ebeb5973391a231 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 29 May 2015 14:23:43 -0500 Subject: [PATCH 0171/2431] Changelog update --- CHANGELOG.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 45b087bc5f..bfa6776afe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,41 @@ +2.6.0 +===== + +This release adds support for Cassandra 2.2 features, including version +4 of the native protocol. + +Features +-------- +* Default load balancing policy to TokenAware(DCAware) (PYTHON-160) +* Configuration option for connection timeout (PYTHON-206) +* Support User Defined Function and Aggregate metadata in C* 2.2 (PYTHON-211) +* Surface request client in QueryTrace for C* 2.2+ (PYTHON-235) +* Implement new request failure messages in protocol v4+ (PYTHON-238) +* Metadata model now maps index meta by index name (PYTHON-241) +* Support new types in C* 2.2: date, time, smallint, tinyint (PYTHON-245, 295) +* cqle: add Double column type and remove Float overload (PYTHON-246) +* Use partition key column information in prepared response for protocol v4+ (PYTHON-277) +* Support message custom payloads in protocol v4+ (PYTHON-280) +* Deprecate refresh_schema and replace with functions for specific entities (PYTHON-291) +* Save trace id even when trace complete times out (PYTHON-302) +* Warn when registering client UDT class for protocol < v3 (PYTHON-305) +* Support client warnings returned with messages in protocol v4+ (PYTHON-315) +* Ability to distinguish between NULL and UNSET values in protocol v4+ (PYTHON-317) +* Expose CQL keywords in API (PYTHON-324) + +Bug Fixes +--------- +* IPv6 address support on Windows (PYTHON-20) +* Convert exceptions during automatic re-preparation to nice exceptions (PYTHON-207) +* cqle: Quote keywords properly in table management functions (PYTHON-244) +* Don't default to GeventConnection when gevent is loaded, but not monkey-patched (PYTHON-289) +* Pass dynamic host from SaslAuthProvider to SaslAuthenticator (PYTHON-300) +* Make protocol read_inet work for Windows (PYTHON-309) +* cqle: Correct encoding for nested types (PYTHON-311) +* Update list of CQL keywords used quoting identifiers (PYTHON-319) +* Make ConstantReconnectionPolicy work with infinite retries (github #327, PYTHON-325) +* Accept UUIDs with uppercase hex as valid in cqlengine (github #335) + 2.5.1 ===== April 23, 2015 From 38705c229ab4c1b4a047f8b140408b1ef456c49f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 29 May 2015 15:09:15 -0500 Subject: [PATCH 0172/2431] cqle: Note about Date type in upgrade guide --- docs/api/cassandra/cqlengine/columns.rst | 2 +- docs/cqlengine/upgrade_guide.rst | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/api/cassandra/cqlengine/columns.rst b/docs/api/cassandra/cqlengine/columns.rst index 7e3957e0cb..ae597a55f7 100644 --- a/docs/api/cassandra/cqlengine/columns.rst +++ b/docs/api/cassandra/cqlengine/columns.rst @@ -52,7 +52,7 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: Counter -.. autoclass:: Date +.. autoclass:: Date(**kwargs) .. autoclass:: DateTime(**kwargs) diff --git a/docs/cqlengine/upgrade_guide.rst b/docs/cqlengine/upgrade_guide.rst index 831f94d7cf..9af5876426 100644 --- a/docs/cqlengine/upgrade_guide.rst +++ b/docs/cqlengine/upgrade_guide.rst @@ -8,13 +8,23 @@ conversion to this package will still require certain minimal updates (namely, i **THERE IS ONE FUNCTIONAL CHANGE**, described in the first section below. -Functional Change -================= +Functional Changes +================== +List Prepend Reversing +---------------------- Legacy cqlengine included a workaround for a Cassandra bug in which prepended list segments were reversed (`CASSANDRA-8733 `_). As of this integration, this workaround is removed. The first released integrated version emits a warning when prepend is used. Subsequent versions will have this warning removed. +Date Column Type +---------------- +The Date column type in legacy cqlengine used a ``timestamp`` CQL type and truncated the time. +Going forward, the :class:`~.columns.Date` type represents a ``date`` for Cassandra 2.2+ +(`PYTHON-245 `_). +Users of the legacy functionality should convert models to use :class:`~.columns.DateTime` (which +uses ``timestamp`` internally), and use the build-in ``datetime.date`` for input values. + Remove cqlengine ================ To avoid confusion or mistakes using the legacy package in your application, it From dc433b91e6d03f2cecce36cebd1a50fd8e0fdf3e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 09:23:10 -0500 Subject: [PATCH 0173/2431] Add comment about dual-use socket struct util.sockaddr --- cassandra/util.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cassandra/util.py b/cassandra/util.py index f188cdcb2f..45ef0e4fe7 100644 --- a/cassandra/util.py +++ b/cassandra/util.py @@ -1024,15 +1024,23 @@ def __str__(self): """ import ctypes - class sockaddr(ctypes.Structure): + """ + Shared struct for ipv4 and ipv6. + + https://msdn.microsoft.com/en-us/library/windows/desktop/ms740496(v=vs.85).aspx + + ``__pad1`` always covers the port. + + When being used for ``sockaddr_in6``, ``ipv4_addr`` actually covers ``sin6_flowinfo``, resulting + in proper alignment for ``ipv6_addr``. + """ _fields_ = [("sa_family", ctypes.c_short), ("__pad1", ctypes.c_ushort), ("ipv4_addr", ctypes.c_byte * 4), ("ipv6_addr", ctypes.c_byte * 16), ("__pad2", ctypes.c_ulong)] - if hasattr(ctypes, 'windll'): WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA @@ -1043,7 +1051,6 @@ def not_windows(*args): WSAStringToAddressA = not_windows WSAAddressToStringA = not_windows - def inet_pton(address_family, ip_string): if address_family == socket.AF_INET: return socket.inet_aton(ip_string) @@ -1066,7 +1073,6 @@ def inet_pton(address_family, ip_string): raise socket.error('unknown address family') - def inet_ntop(address_family, packed_ip): if address_family == socket.AF_INET: return socket.inet_ntoa(packed_ip) From 6fec9d17a4ab91a15ad6f86ddddd450573f725ae Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 12:52:39 -0500 Subject: [PATCH 0174/2431] Add Cluster.connect_timeout to api doc --- docs/api/cassandra/cluster.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 1e4910cc0c..2b2413354f 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -47,6 +47,8 @@ .. autoattribute:: topology_event_refresh_window + .. autoattribute:: connect_timeout + .. automethod:: connect .. automethod:: shutdown From 2b7d9bc8ab36e0b806476451a40e8b70b7ee0b19 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 13:20:21 -0500 Subject: [PATCH 0175/2431] Add cluster.ResponseFuture.warnings to api doc --- docs/api/cassandra/cluster.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 2b2413354f..195d03a6ef 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -128,6 +128,8 @@ .. autoattribute:: has_more_pages + .. autoattribute:: warnings + .. automethod:: start_fetching_next_page() .. automethod:: add_callback(fn, *args, **kwargs) From a15729c74c2a3982fdccf0501b144a2614e4e173 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 13:20:48 -0500 Subject: [PATCH 0176/2431] cqle: Tighten-up columns class docs with inheritied init --- docs/api/cassandra/cqlengine/columns.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/cassandra/cqlengine/columns.rst b/docs/api/cassandra/cqlengine/columns.rst index ae597a55f7..7c4695d323 100644 --- a/docs/api/cassandra/cqlengine/columns.rst +++ b/docs/api/cassandra/cqlengine/columns.rst @@ -70,15 +70,15 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: Set -.. autoclass:: SmallInt +.. autoclass:: SmallInt(**kwargs) .. autoclass:: Text -.. autoclass:: Time +.. autoclass:: Time(**kwargs) .. autoclass:: TimeUUID(**kwargs) -.. autoclass:: TinyInt +.. autoclass:: TinyInt(**kwargs) .. autoclass:: UserDefinedType From c967f10fc201b44743526455d69c79a1df6284ec Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 13:49:10 -0500 Subject: [PATCH 0177/2431] Update CHANGELOG.rst --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bfa6776afe..63f4185c30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,4 +1,4 @@ -2.6.0 +2.6.0rc1 ===== This release adds support for Cassandra 2.2 features, including version From 03c2b2ae1f5abbd8ea0383130ada2db1f40f86c8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 15:44:26 -0500 Subject: [PATCH 0178/2431] Update README.rst --- README.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index bdc6dca7eb..261d1e31a9 100644 --- a/README.rst +++ b/README.rst @@ -4,16 +4,26 @@ DataStax Python Driver for Apache Cassandra .. image:: https://travis-ci.org/datastax/python-driver.png?branch=master :target: https://travis-ci.org/datastax/python-driver -A Python client driver for Apache Cassandra. This driver works exclusively -with the Cassandra Query Language v3 (CQL3) and Cassandra's native -protocol. Cassandra versions 1.2 through 2.1 are supported. +A modern, `feature-rich `_ and highly-tunable Python client library for Apache Cassandra (1.2+) and DataStax Enterprise (3.1+) using exclusively Cassandra's binary protocol and Cassandra Query Language v3. The driver supports Python 2.6, 2.7, 3.3, and 3.4*. -* cqlengine component presently supports Python 2.7+ +\* cqlengine component presently supports Python 2.7+ **Help us focus our efforts!** Provide your input on the `Platform and Runtime Survey `_. +Features +-------- +* `Synchronous `_ and `Asynchronous `_ APIs +* `Simple, Prepared, and Batch statements `_ +* Asynchronous IO, parallel execution, request pipelining +* `Connection pooling `_ +* Automatic node discovery +* `Automatic reconnection `_ +* Configurable `load balancing `_ and `retry policies `_ +* `Concurrent execution utilities `_ +* `Object mapper `_ + Installation ------------ Installation through pip is recommended:: From c050a96bc711006ee33cec24ca13c84687a8420b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 1 Jun 2015 15:46:49 -0500 Subject: [PATCH 0179/2431] Update README.rst --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 261d1e31a9..ce818a3b57 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,9 @@ The driver supports Python 2.6, 2.7, 3.3, and 3.4*. \* cqlengine component presently supports Python 2.7+ -**Help us focus our efforts!** Provide your input on the `Platform and Runtime Survey `_. +Feedback Requested +------------------ +**Help us focus our efforts!** Provide your input on the `Platform and Runtime Survey `_ (we kept it short). Features -------- From f7fac976d875304b92a9894f7e85a70ffc2f7fbb Mon Sep 17 00:00:00 2001 From: jpuerta Date: Mon, 1 Jun 2015 10:42:47 +0100 Subject: [PATCH 0180/2431] cqle: backport cqlengine to support Python 2.6 https://datastax-oss.atlassian.net/browse/PYTHON-288 Add Python 2.6 compliancy: - Remove dict/set comprehensions. - Use cassandra's OrderedDict. - Replace str.format() implicit placeholders ('{}') with explicit ones ('{0} {1} ...'). - Implement cqlengine.functions.get_total_seconds() as alternative to timedelta.total_seconds(). cqlengine integration tests remain to be backported. --- README.rst | 2 +- cassandra/cqlengine/columns.py | 71 ++++++++++++++++--------------- cassandra/cqlengine/functions.py | 22 +++++++--- cassandra/cqlengine/management.py | 48 ++++++++++----------- cassandra/cqlengine/models.py | 38 ++++++++--------- cassandra/cqlengine/named.py | 4 +- cassandra/cqlengine/operators.py | 2 +- cassandra/cqlengine/query.py | 32 +++++++------- cassandra/cqlengine/statements.py | 56 ++++++++++++------------ cassandra/cqlengine/usertype.py | 6 +-- 10 files changed, 145 insertions(+), 136 deletions(-) diff --git a/README.rst b/README.rst index ce818a3b57..110505da97 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ A modern, `feature-rich `_ a The driver supports Python 2.6, 2.7, 3.3, and 3.4*. -\* cqlengine component presently supports Python 2.7+ +\* cqlengine component presently supports Python 2.6+ Feedback Requested ------------------ diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index d418b11947..b5c49ecea6 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -21,6 +21,7 @@ from cassandra import util from cassandra.cqltypes import DateType, SimpleDateType from cassandra.cqlengine import ValidationError +from cassandra.cqlengine.functions import get_total_seconds log = logging.getLogger(__name__) @@ -186,7 +187,7 @@ def validate(self, value): """ if value is None: if self.required: - raise ValidationError('{} - None values are not allowed'.format(self.column_name or self.db_field)) + raise ValidationError('{0} - None values are not allowed'.format(self.column_name or self.db_field)) return value def to_python(self, value): @@ -228,7 +229,7 @@ def get_column_def(self): Returns a column definition for CQL table definition """ static = "static" if self.static else "" - return '{} {} {}'.format(self.cql, self.db_type, static) + return '{0} {1} {2}'.format(self.cql, self.db_type, static) # TODO: make columns use cqltypes under the hood # until then, this bridges the gap in using types along with cassandra.metadata for CQL generation @@ -250,14 +251,14 @@ def db_field_name(self): @property def db_index_name(self): """ Returns the name of the cql index """ - return 'index_{}'.format(self.db_field_name) + return 'index_{0}'.format(self.db_field_name) @property def cql(self): return self.get_cql() def get_cql(self): - return '"{}"'.format(self.db_field_name) + return '"{0}"'.format(self.db_field_name) def _val_is_null(self, val): """ determines if the given value equates to a null value for the given column type """ @@ -323,13 +324,13 @@ def validate(self, value): if value is None: return if not isinstance(value, (six.string_types, bytearray)) and value is not None: - raise ValidationError('{} {} is not a string'.format(self.column_name, type(value))) + raise ValidationError('{0} {1} is not a string'.format(self.column_name, type(value))) if self.max_length: if len(value) > self.max_length: - raise ValidationError('{} is longer than {} characters'.format(self.column_name, self.max_length)) + raise ValidationError('{0} is longer than {1} characters'.format(self.column_name, self.max_length)) if self.min_length: if len(value) < self.min_length: - raise ValidationError('{} is shorter than {} characters'.format(self.column_name, self.min_length)) + raise ValidationError('{0} is shorter than {1} characters'.format(self.column_name, self.min_length)) return value @@ -347,7 +348,7 @@ def validate(self, value): try: return int(val) except (TypeError, ValueError): - raise ValidationError("{} {} can't be converted to integral value".format(self.column_name, value)) + raise ValidationError("{0} {1} can't be converted to integral value".format(self.column_name, value)) def to_python(self, value): return self.validate(value) @@ -399,7 +400,7 @@ def validate(self, value): return int(val) except (TypeError, ValueError): raise ValidationError( - "{} {} can't be converted to integral value".format(self.column_name, value)) + "{0} {1} can't be converted to integral value".format(self.column_name, value)) def to_python(self, value): return self.validate(value) @@ -463,11 +464,11 @@ def to_database(self, value): if isinstance(value, date): value = datetime(value.year, value.month, value.day) else: - raise ValidationError("{} '{}' is not a datetime object".format(self.column_name, value)) + raise ValidationError("{0} '{1}' is not a datetime object".format(self.column_name, value)) epoch = datetime(1970, 1, 1, tzinfo=value.tzinfo) - offset = epoch.tzinfo.utcoffset(epoch).total_seconds() if epoch.tzinfo else 0 + offset = get_total_seconds(epoch.tzinfo.utcoffset(epoch)) if epoch.tzinfo else 0 - return int(((value - epoch).total_seconds() - offset) * 1000) + return int((get_total_seconds(value - epoch) - offset) * 1000) class Date(Column): @@ -530,7 +531,7 @@ def validate(self, value): except ValueError: # fall-through to error pass - raise ValidationError("{} {} is not a valid uuid".format( + raise ValidationError("{0} {1} is not a valid uuid".format( self.column_name, value)) def to_python(self, value): @@ -561,8 +562,8 @@ def from_datetime(self, dt): global _last_timestamp epoch = datetime(1970, 1, 1, tzinfo=dt.tzinfo) - offset = epoch.tzinfo.utcoffset(epoch).total_seconds() if epoch.tzinfo else 0 - timestamp = (dt - epoch).total_seconds() - offset + offset = get_total_seconds(epoch.tzinfo.utcoffset(epoch)) if epoch.tzinfo else 0 + timestamp = get_total_seconds(dt - epoch) - offset node = None clock_seq = None @@ -611,7 +612,7 @@ def validate(self, value): try: return float(value) except (TypeError, ValueError): - raise ValidationError("{} {} is not a valid float".format(self.column_name, value)) + raise ValidationError("{0} {1} is not a valid float".format(self.column_name, value)) def to_python(self, value): return self.validate(value) @@ -660,7 +661,7 @@ def validate(self, value): try: return _Decimal(val) except InvalidOperation: - raise ValidationError("{} '{}' can't be coerced to decimal".format(self.column_name, val)) + raise ValidationError("{0} '{1}' can't be coerced to decimal".format(self.column_name, val)) def to_python(self, value): return self.validate(value) @@ -702,7 +703,7 @@ def validate(self, value): # It is dangerous to let collections have more than 65535. # See: https://issues.apache.org/jira/browse/CASSANDRA-5428 if value is not None and len(value) > 65535: - raise ValidationError("{} Collection can't have more than 65535 elements.".format(self.column_name)) + raise ValidationError("{0} Collection can't have more than 65535 elements.".format(self.column_name)) return value def _val_is_null(self, val): @@ -726,7 +727,7 @@ def __init__(self, value_type, strict=True, default=set, **kwargs): type on validation, or raise a validation error, defaults to True """ self.strict = strict - self.db_type = 'set<{}>'.format(value_type.db_type) + self.db_type = 'set<{0}>'.format(value_type.db_type) super(Set, self).__init__(value_type, default=default, **kwargs) def validate(self, value): @@ -736,24 +737,24 @@ def validate(self, value): types = (set,) if self.strict else (set, list, tuple) if not isinstance(val, types): if self.strict: - raise ValidationError('{} {} is not a set object'.format(self.column_name, val)) + raise ValidationError('{0} {1} is not a set object'.format(self.column_name, val)) else: - raise ValidationError('{} {} cannot be coerced to a set object'.format(self.column_name, val)) + raise ValidationError('{0} {1} cannot be coerced to a set object'.format(self.column_name, val)) if None in val: - raise ValidationError("{} None not allowed in a set".format(self.column_name)) + raise ValidationError("{0} None not allowed in a set".format(self.column_name)) - return {self.value_col.validate(v) for v in val} + return set(self.value_col.validate(v) for v in val) def to_python(self, value): if value is None: return set() - return {self.value_col.to_python(v) for v in value} + return set(self.value_col.to_python(v) for v in value) def to_database(self, value): if value is None: return None - return {self.value_col.to_database(v) for v in value} + return set(self.value_col.to_database(v) for v in value) class List(BaseContainerColumn): @@ -766,7 +767,7 @@ def __init__(self, value_type, default=list, **kwargs): """ :param value_type: a column class indicating the types of the value """ - self.db_type = 'list<{}>'.format(value_type.db_type) + self.db_type = 'list<{0}>'.format(value_type.db_type) return super(List, self).__init__(value_type=value_type, default=default, **kwargs) def validate(self, value): @@ -774,9 +775,9 @@ def validate(self, value): if val is None: return if not isinstance(val, (set, list, tuple)): - raise ValidationError('{} {} is not a list object'.format(self.column_name, val)) + raise ValidationError('{0} {1} is not a list object'.format(self.column_name, val)) if None in val: - raise ValidationError("{} None is not allowed in a list".format(self.column_name)) + raise ValidationError("{0} None is not allowed in a list".format(self.column_name)) return [self.value_col.validate(v) for v in val] def to_python(self, value): @@ -802,7 +803,7 @@ def __init__(self, key_type, value_type, default=dict, **kwargs): :param value_type: a column class indicating the types of the value """ - self.db_type = 'map<{}, {}>'.format(key_type.db_type, value_type.db_type) + self.db_type = 'map<{0}, {1}>'.format(key_type.db_type, value_type.db_type) inheritance_comparator = issubclass if isinstance(key_type, type) else isinstance if not inheritance_comparator(key_type, Column): @@ -825,21 +826,21 @@ def validate(self, value): if val is None: return if not isinstance(val, dict): - raise ValidationError('{} {} is not a dict object'.format(self.column_name, val)) + raise ValidationError('{0} {1} is not a dict object'.format(self.column_name, val)) if None in val: raise ValidationError("{} None is not allowed in a map".format(self.column_name)) - return {self.key_col.validate(k): self.value_col.validate(v) for k, v in val.items()} + return dict((self.key_col.validate(k), self.value_col.validate(v)) for k, v in val.items()) def to_python(self, value): if value is None: return {} if value is not None: - return {self.key_col.to_python(k): self.value_col.to_python(v) for k, v in value.items()} + return dict((self.key_col.to_python(k), self.value_col.to_python(v)) for k, v in value.items()) def to_database(self, value): if value is None: return None - return {self.key_col.to_database(k): self.value_col.to_database(v) for k, v in value.items()} + return dict((self.key_col.to_database(k), self.value_col.to_database(v)) for k, v in value.items()) @property def sub_columns(self): @@ -901,7 +902,7 @@ def __init__(self, model): @property def db_field_name(self): - return 'token({})'.format(', '.join(['"{}"'.format(c.db_field_name) for c in self.partition_columns])) + return 'token({0})'.format(', '.join(['"{0}"'.format(c.db_field_name) for c in self.partition_columns])) def to_database(self, value): from cqlengine.functions import Token @@ -910,4 +911,4 @@ def to_database(self, value): return value def get_cql(self): - return "token({})".format(", ".join(c.cql for c in self.partition_columns)) + return "token({0})".format(", ".join(c.cql for c in self.partition_columns)) diff --git a/cassandra/cqlengine/functions.py b/cassandra/cqlengine/functions.py index 5c6e4c7cd6..43c98afea5 100644 --- a/cassandra/cqlengine/functions.py +++ b/cassandra/cqlengine/functions.py @@ -16,6 +16,14 @@ from cassandra.cqlengine import UnicodeMixin, ValidationError +import sys + +if sys.version_info >= (2, 7): + def get_total_seconds(td): + return td.total_seconds() +else: + def get_total_seconds(td): + return 86400*td.days + td.seconds + td.microseconds/1e6 class QueryValue(UnicodeMixin): """ @@ -23,7 +31,7 @@ class QueryValue(UnicodeMixin): be passed into .filter() keyword args """ - format_string = '%({})s' + format_string = '%({0})s' def __init__(self, value): self.value = value @@ -58,7 +66,7 @@ class MinTimeUUID(BaseQueryFunction): http://cassandra.apache.org/doc/cql3/CQL.html#timeuuidFun """ - format_string = 'MinTimeUUID(%({})s)' + format_string = 'MinTimeUUID(%({0})s)' def __init__(self, value): """ @@ -71,8 +79,8 @@ def __init__(self, value): def to_database(self, val): epoch = datetime(1970, 1, 1, tzinfo=val.tzinfo) - offset = epoch.tzinfo.utcoffset(epoch).total_seconds() if epoch.tzinfo else 0 - return int(((val - epoch).total_seconds() - offset) * 1000) + offset = get_total_seconds(epoch.tzinfo.utcoffset(epoch)) if epoch.tzinfo else 0 + return int((get_total_seconds(val - epoch) - offset) * 1000) def update_context(self, ctx): ctx[str(self.context_id)] = self.to_database(self.value) @@ -85,7 +93,7 @@ class MaxTimeUUID(BaseQueryFunction): http://cassandra.apache.org/doc/cql3/CQL.html#timeuuidFun """ - format_string = 'MaxTimeUUID(%({})s)' + format_string = 'MaxTimeUUID(%({0})s)' def __init__(self, value): """ @@ -125,8 +133,8 @@ def get_context_size(self): return len(self.value) def __unicode__(self): - token_args = ', '.join('%({})s'.format(self.context_id + i) for i in range(self.get_context_size())) - return "token({})".format(token_args) + token_args = ', '.join('%({0})s'.format(self.context_id + i) for i in range(self.get_context_size())) + return "token({0})".format(token_args) def update_context(self, ctx): for i, (col, val) in enumerate(zip(self._columns, self.value)): diff --git a/cassandra/cqlengine/management.py b/cassandra/cqlengine/management.py index 5d5e641459..41d714107b 100644 --- a/cassandra/cqlengine/management.py +++ b/cassandra/cqlengine/management.py @@ -79,12 +79,12 @@ def create_keyspace(name, strategy_class, replication_factor, durable_writes=Tru replication_map.pop('replication_factor', None) query = """ - CREATE KEYSPACE {} - WITH REPLICATION = {} + CREATE KEYSPACE {0} + WITH REPLICATION = {1} """.format(metadata.protect_name(name), json.dumps(replication_map).replace('"', "'")) if strategy_class != 'SimpleStrategy': - query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false') + query += " AND DURABLE_WRITES = {0}".format('true' if durable_writes else 'false') execute(query) @@ -163,7 +163,7 @@ def drop_keyspace(name): cluster = get_cluster() if name in cluster.metadata.keyspaces: - execute("DROP KEYSPACE {}".format(metadata.protect_name(name))) + execute("DROP KEYSPACE {0}".format(metadata.protect_name(name))) def sync_table(model): @@ -235,7 +235,7 @@ def sync_table(model): continue # skip columns already defined # add missing column using the column def - query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def()) + query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def()) execute(query) db_fields_not_in_model = model_fields.symmetric_difference(field_names) @@ -252,9 +252,9 @@ def sync_table(model): if table.columns[column.db_field_name].index: continue - qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)] - qs += ['ON {}'.format(cf_name)] - qs += ['("{}")'.format(column.db_field_name)] + qs = ['CREATE INDEX index_{0}_{1}'.format(raw_cf_name, column.db_field_name)] + qs += ['ON {0}'.format(cf_name)] + qs += ['("{0}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs) @@ -310,7 +310,7 @@ def _sync_type(ks_name, type_model, omit_subtypes=None): for field in type_model._fields.values(): model_fields.add(field.db_field_name) if field.db_field_name not in defined_fields: - execute("ALTER TYPE {} ADD {}".format(type_name_qualified, field.get_column_def())) + execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def())) type_model.register_for_keyspace(ks_name) @@ -333,7 +333,7 @@ def get_create_type(type_model, keyspace): def get_create_table(model): cf_name = model.column_family_name() - qs = ['CREATE TABLE {}'.format(cf_name)] + qs = ['CREATE TABLE {0}'.format(cf_name)] # add column types pkeys = [] # primary keys @@ -344,15 +344,15 @@ def add_column(col): s = col.get_column_def() if col.primary_key: keys = (pkeys if col.partition_key else ckeys) - keys.append('"{}"'.format(col.db_field_name)) + keys.append('"{0}"'.format(col.db_field_name)) qtypes.append(s) for name, col in model._columns.items(): add_column(col) - qtypes.append('PRIMARY KEY (({}){})'.format(', '.join(pkeys), ckeys and ', ' + ', '.join(ckeys) or '')) + qtypes.append('PRIMARY KEY (({0}){1})'.format(', '.join(pkeys), ckeys and ', ' + ', '.join(ckeys) or '')) - qs += ['({})'.format(', '.join(qtypes))] + qs += ['({0})'.format(', '.join(qtypes))] with_qs = [] @@ -361,25 +361,25 @@ def add_column(col): 'index_interval', 'memtable_flush_period_in_ms', 'populate_io_cache_on_flush', 'read_repair_chance', 'replicate_on_write'] for prop_name in table_properties: - prop_value = getattr(model, '__{}__'.format(prop_name), None) + prop_value = getattr(model, '__{0}__'.format(prop_name), None) if prop_value is not None: # Strings needs to be single quoted if isinstance(prop_value, six.string_types): - prop_value = "'{}'".format(prop_value) - with_qs.append("{} = {}".format(prop_name, prop_value)) + prop_value = "'{0}'".format(prop_value) + with_qs.append("{0} = {1}".format(prop_name, prop_value)) - _order = ['"{}" {}'.format(c.db_field_name, c.clustering_order or 'ASC') for c in model._clustering_keys.values()] + _order = ['"{0}" {1}'.format(c.db_field_name, c.clustering_order or 'ASC') for c in model._clustering_keys.values()] if _order: - with_qs.append('clustering order by ({})'.format(', '.join(_order))) + with_qs.append('clustering order by ({0})'.format(', '.join(_order))) compaction_options = get_compaction_options(model) if compaction_options: compaction_options = json.dumps(compaction_options).replace('"', "'") - with_qs.append("compaction = {}".format(compaction_options)) + with_qs.append("compaction = {0}".format(compaction_options)) # Add table properties. if with_qs: - qs += ['WITH {}'.format(' AND '.join(with_qs))] + qs += ['WITH {0}'.format(' AND '.join(with_qs))] qs = ' '.join(qs) return qs @@ -405,10 +405,10 @@ def setter(key, limited_to_strategy=None): :param limited_to_strategy: SizeTieredCompactionStrategy, LeveledCompactionStrategy :return: """ - mkey = "__compaction_{}__".format(key) + mkey = "__compaction_{0}__".format(key) tmp = getattr(model, mkey) if tmp and limited_to_strategy and limited_to_strategy != model.__compaction__: - raise CQLEngineException("{} is limited to {}".format(key, limited_to_strategy)) + raise CQLEngineException("{0} is limited to {1}".format(key, limited_to_strategy)) if tmp: # Explicitly cast the values to strings to be able to compare the @@ -496,7 +496,7 @@ def update_compaction(model): # jsonify options = json.dumps(options).replace('"', "'") cf_name = model.column_family_name() - query = "ALTER TABLE {} with compaction = {}".format(cf_name, options) + query = "ALTER TABLE {0} with compaction = {1}".format(cf_name, options) execute(query) return True @@ -523,7 +523,7 @@ def drop_table(model): try: meta.keyspaces[ks_name].tables[raw_cf_name] - execute('DROP TABLE {};'.format(model.column_family_name())) + execute('DROP TABLE {0};'.format(model.column_family_name())) except KeyError: pass diff --git a/cassandra/cqlengine/models.py b/cassandra/cqlengine/models.py index 4f6d120250..b0f86d990e 100644 --- a/cassandra/cqlengine/models.py +++ b/cassandra/cqlengine/models.py @@ -280,7 +280,7 @@ def __delete__(self, instance): if self.column.can_delete: instance._values[self.column.column_name].delval() else: - raise AttributeError('cannot delete {} columns'.format(self.column.column_name)) + raise AttributeError('cannot delete {0} columns'.format(self.column.column_name)) class BaseModel(object): @@ -378,8 +378,8 @@ def __init__(self, **values): self._timeout = connection.NOT_SET def __repr__(self): - return '{}({})'.format(self.__class__.__name__, - ', '.join('{}={!r}'.format(k, getattr(self, k)) + return '{0}({1})'.format(self.__class__.__name__, + ', '.join('{0}={1!r}'.format(k, getattr(self, k)) for k in self._defined_columns.keys() if k != self._discriminator_column_name)) @@ -387,8 +387,8 @@ def __str__(self): """ Pretty printing of models by their primary key """ - return '{} <{}>'.format(self.__class__.__name__, - ', '.join('{}={}'.format(k, getattr(self, k)) for k in self._primary_keys.keys())) + return '{0} <{1}>'.format(self.__class__.__name__, + ', '.join('{0}={1}'.format(k, getattr(self, k)) for k in self._primary_keys.keys())) @classmethod def _discover_polymorphic_submodels(cls): @@ -433,16 +433,16 @@ def _construct_instance(cls, values): poly_base._discover_polymorphic_submodels() klass = poly_base._get_model_by_discriminator_value(disc_key) if klass is None: - raise PolymorphicModelException( - 'unrecognized discriminator column {} for class {}'.format(disc_key, poly_base.__name__) + raise PolyMorphicModelException( + 'unrecognized polymorphic key {0} for class {1}'.format(poly_key, poly_base.__name__) ) if not issubclass(klass, cls): - raise PolymorphicModelException( - '{} is not a subclass of {}'.format(klass.__name__, cls.__name__) + raise PolyMorphicModelException( + '{0} is not a subclass of {1}'.format(klass.__name__, cls.__name__) ) - field_dict = {k: v for k, v in field_dict.items() if k in klass._columns.keys()} + field_dict = dict((k, v) for (k, v) in field_dict.items() if k in klass._columns.keys()) else: klass = cls @@ -509,7 +509,7 @@ def column_family_name(cls, include_keyspace=True): """ cf_name = protect_name(cls._raw_column_family_name()) if include_keyspace: - return '{}.{}'.format(protect_name(cls._get_keyspace()), cf_name) + return '{0}.{1}'.format(protect_name(cls._get_keyspace()), cf_name) return cf_name @@ -524,7 +524,7 @@ def _raw_column_family_name(cls): cls._table_name = cls._polymorphic_base._raw_column_family_name() else: camelcase = re.compile(r'([a-z])([A-Z])') - ccase = lambda s: camelcase.sub(lambda v: '{}_{}'.format(v.group(1), v.group(2).lower()), s) + ccase = lambda s: camelcase.sub(lambda v: '{0}_{1}'.format(v.group(1), v.group(2).lower()), s) cf_name = ccase(cls.__name__) # trim to less than 48 characters or cassandra will complain @@ -608,7 +608,7 @@ def create(cls, **kwargs): """ extra_columns = set(kwargs.keys()) - set(cls._columns.keys()) if extra_columns: - raise ValidationError("Incorrect columns passed: {}".format(extra_columns)) + raise ValidationError("Incorrect columns passed: {0}".format(extra_columns)) return cls.objects.create(**kwargs) @classmethod @@ -701,11 +701,11 @@ def update(self, **values): # check for nonexistant columns if col is None: - raise ValidationError("{}.{} has no column named: {}".format(self.__module__, self.__class__.__name__, k)) + raise ValidationError("{0}.{1} has no column named: {2}".format(self.__module__, self.__class__.__name__, k)) # check for primary key update attempts if col.is_primary_key: - raise ValidationError("Cannot apply update to primary key '{}' for {}.{}".format(k, self.__module__, self.__class__.__name__)) + raise ValidationError("Cannot apply update to primary key '{0}' for {1}.{2}".format(k, self.__module__, self.__class__.__name__)) setattr(self, k, v) @@ -808,7 +808,7 @@ def _transform_column(col_name, col_obj): discriminator_columns = [c for c in column_definitions if c[1].discriminator_column] is_polymorphic = len(discriminator_columns) > 0 if len(discriminator_columns) > 1: - raise ModelDefinitionException('only one discriminator_column (polymorphic_key (deprecated)) can be defined in a model, {} found'.format(len(discriminator_columns))) + raise ModelDefinitionException('only one discriminator_column (polymorphic_key (deprecated)) can be defined in a model, {0} found'.format(len(discriminator_columns))) if attrs['__discriminator_value__'] and not is_polymorphic: raise ModelDefinitionException('__discriminator_value__ specified, but no base columns defined with discriminator_column=True') @@ -847,7 +847,7 @@ def _get_polymorphic_base(bases): for k, v in column_definitions: # don't allow a column with the same name as a built-in attribute or method if k in BaseModel.__dict__: - raise ModelDefinitionException("column '{}' conflicts with built-in attribute/method".format(k)) + raise ModelDefinitionException("column '{0}' conflicts with built-in attribute/method".format(k)) # counter column primary keys are not allowed if (v.primary_key or v.partition_key) and isinstance(v, (columns.Counter, columns.BaseContainerColumn)): @@ -881,11 +881,11 @@ def _get_polymorphic_base(bases): for v in column_dict.values(): # check for duplicate column names if v.db_field_name in col_names: - raise ModelException("{} defines the column {} more than once".format(name, v.db_field_name)) + raise ModelException("{0} defines the column {1} more than once".format(name, v.db_field_name)) if v.clustering_order and not (v.primary_key and not v.partition_key): raise ModelException("clustering_order may be specified only for clustering primary keys") if v.clustering_order and v.clustering_order.lower() not in ('asc', 'desc'): - raise ModelException("invalid clustering order {} for column {}".format(repr(v.clustering_order), v.db_field_name)) + raise ModelException("invalid clustering order {0} for column {1}".format(repr(v.clustering_order), v.db_field_name)) col_names.add(v.db_field_name) # create db_name -> model name map for loading diff --git a/cassandra/cqlengine/named.py b/cassandra/cqlengine/named.py index b38c07820e..7bc229311d 100644 --- a/cassandra/cqlengine/named.py +++ b/cassandra/cqlengine/named.py @@ -63,7 +63,7 @@ def cql(self): return self.get_cql() def get_cql(self): - return '"{}"'.format(self.name) + return '"{0}"'.format(self.name) def to_database(self, val): return val @@ -97,7 +97,7 @@ def column_family_name(self, include_keyspace=True): otherwise, it creates it from the module and class name """ if include_keyspace: - return '{}.{}'.format(self.keyspace, self.name) + return '{0}.{1}'.format(self.keyspace, self.name) else: return self.name diff --git a/cassandra/cqlengine/operators.py b/cassandra/cqlengine/operators.py index 08c8ccb318..5ddbfed83b 100644 --- a/cassandra/cqlengine/operators.py +++ b/cassandra/cqlengine/operators.py @@ -50,7 +50,7 @@ def _recurse(klass): try: return cls.opmap[symbol.upper()] except KeyError: - raise QueryOperatorException("{} doesn't map to a QueryOperator".format(symbol)) + raise QueryOperatorException("{0} doesn't map to a QueryOperator".format(symbol)) class BaseWhereOperator(BaseQueryOperator): diff --git a/cassandra/cqlengine/query.py b/cassandra/cqlengine/query.py index 2a5c9744dc..19b06552f9 100644 --- a/cassandra/cqlengine/query.py +++ b/cassandra/cqlengine/query.py @@ -174,7 +174,7 @@ def add_callback(self, fn, *args, **kwargs): :param **kwargs: Named arguments to be passed to the callback at the time of execution """ if not callable(fn): - raise ValueError("Value for argument 'fn' is {} and is not a callable object.".format(type(fn))) + raise ValueError("Value for argument 'fn' is {0} and is not a callable object.".format(type(fn))) self._callbacks.append((fn, args, kwargs)) def execute(self): @@ -197,7 +197,7 @@ def execute(self): else: raise ValueError("Batch expects a long, a timedelta, or a datetime") - opener += ' USING TIMESTAMP {}'.format(ts) + opener += ' USING TIMESTAMP {0}'.format(ts) query_list = [opener] parameters = {} @@ -453,7 +453,7 @@ def _parse_filter_arg(self, arg): elif len(statement) == 2: return statement[0], statement[1] else: - raise QueryException("Can't parse '{}'".format(arg)) + raise QueryException("Can't parse '{0}'".format(arg)) def iff(self, *args, **kwargs): """Adds IF statements to queryset""" @@ -463,7 +463,7 @@ def iff(self, *args, **kwargs): clone = copy.deepcopy(self) for operator in args: if not isinstance(operator, TransactionClause): - raise QueryException('{} is not a valid query operator'.format(operator)) + raise QueryException('{0} is not a valid query operator'.format(operator)) clone._transaction.append(operator) for col_name, val in kwargs.items(): @@ -476,7 +476,7 @@ def iff(self, *args, **kwargs): raise QueryException("Virtual column 'pk__token' may only be compared to Token() values") column = columns._PartitionKeysToken(self.model) else: - raise QueryException("Can't resolve column name: '{}'".format(col_name)) + raise QueryException("Can't resolve column name: '{0}'".format(col_name)) if isinstance(val, Token): if col_name != 'pk__token': @@ -484,7 +484,7 @@ def iff(self, *args, **kwargs): partition_columns = column.partition_columns if len(partition_columns) != len(val.value): raise QueryException( - 'Token() received {} arguments but model has {} partition keys'.format( + 'Token() received {0} arguments but model has {1} partition keys'.format( len(val.value), len(partition_columns))) val.set_columns(partition_columns) @@ -512,7 +512,7 @@ def filter(self, *args, **kwargs): clone = copy.deepcopy(self) for operator in args: if not isinstance(operator, WhereClause): - raise QueryException('{} is not a valid query operator'.format(operator)) + raise QueryException('{0} is not a valid query operator'.format(operator)) clone._where.append(operator) for arg, val in kwargs.items(): @@ -528,7 +528,7 @@ def filter(self, *args, **kwargs): column = columns._PartitionKeysToken(self.model) quote_field = False else: - raise QueryException("Can't resolve column name: '{}'".format(col_name)) + raise QueryException("Can't resolve column name: '{0}'".format(col_name)) if isinstance(val, Token): if col_name != 'pk__token': @@ -536,7 +536,7 @@ def filter(self, *args, **kwargs): partition_columns = column.partition_columns if len(partition_columns) != len(val.value): raise QueryException( - 'Token() received {} arguments but model has {} partition keys'.format( + 'Token() received {0} arguments but model has {1} partition keys'.format( len(val.value), len(partition_columns))) val.set_columns(partition_columns) @@ -580,7 +580,7 @@ def get(self, *args, **kwargs): if len(self._result_cache) == 0: raise self.model.DoesNotExist elif len(self._result_cache) > 1: - raise self.model.MultipleObjectsReturned('{} objects found'.format(len(self._result_cache))) + raise self.model.MultipleObjectsReturned('{0} objects found'.format(len(self._result_cache))) else: return self[0] @@ -628,7 +628,7 @@ class Comment(Model): conditions = [] for colname in colnames: - conditions.append('"{}" {}'.format(*self._get_ordering_condition(colname))) + conditions.append('"{0}" {1}'.format(*self._get_ordering_condition(colname))) clone = copy.deepcopy(self) clone._order.extend(conditions) @@ -689,7 +689,7 @@ def _only_or_defer(self, action, fields): missing_fields = [f for f in fields if f not in self.model._columns.keys()] if missing_fields: raise QueryException( - "Can't resolve fields {} in {}".format( + "Can't resolve fields {0} in {1}".format( ', '.join(missing_fields), self.model.__name__)) if action == 'defer': @@ -821,12 +821,12 @@ def _get_ordering_condition(self, colname): column = self.model._columns.get(colname) if column is None: - raise QueryException("Can't resolve the column name: '{}'".format(colname)) + raise QueryException("Can't resolve the column name: '{0}'".format(colname)) # validate the column selection if not column.primary_key: raise QueryException( - "Can't order on '{}', can only order on (clustered) primary keys".format(colname)) + "Can't order on '{0}', can only order on (clustered) primary keys".format(colname)) pks = [v for k, v in self.model._columns.items() if v.primary_key] if column == pks[0]: @@ -971,10 +971,10 @@ class Row(Model): col = self.model._columns.get(col_name) # check for nonexistant columns if col is None: - raise ValidationError("{}.{} has no column named: {}".format(self.__module__, self.model.__name__, col_name)) + raise ValidationError("{0}.{1} has no column named: {2}".format(self.__module__, self.model.__name__, col_name)) # check for primary key update attempts if col.is_primary_key: - raise ValidationError("Cannot apply update to primary key '{}' for {}.{}".format(col_name, self.__module__, self.model.__name__)) + raise ValidationError("Cannot apply update to primary key '{0}' for {1}.{2}".format(col_name, self.__module__, self.model.__name__)) # we should not provide default values in this use case. val = col.validate(val) diff --git a/cassandra/cqlengine/statements.py b/cassandra/cqlengine/statements.py index 866f6f2958..c1a7a51130 100644 --- a/cassandra/cqlengine/statements.py +++ b/cassandra/cqlengine/statements.py @@ -108,7 +108,7 @@ def __init__(self, field, operator, value, quote_field=True): """ if not isinstance(operator, BaseWhereOperator): raise StatementException( - "operator must be of type {}, got {}".format(BaseWhereOperator, type(operator)) + "operator must be of type {0}, got {1}".format(BaseWhereOperator, type(operator)) ) super(WhereClause, self).__init__(field, value) self.operator = operator @@ -116,8 +116,8 @@ def __init__(self, field, operator, value, quote_field=True): self.quote_field = quote_field def __unicode__(self): - field = ('"{}"' if self.quote_field else '{}').format(self.field) - return u'{} {} {}'.format(field, self.operator, six.text_type(self.query_value)) + field = ('"{0}"' if self.quote_field else '{0}').format(self.field) + return u'{0} {1} {2}'.format(field, self.operator, six.text_type(self.query_value)) def __hash__(self): return super(WhereClause, self).__hash__() ^ hash(self.operator) @@ -145,7 +145,7 @@ class AssignmentClause(BaseClause): """ a single variable st statement """ def __unicode__(self): - return u'"{}" = %({})s'.format(self.field, self.context_id) + return u'"{0}" = %({1})s'.format(self.field, self.context_id) def insert_tuple(self): return self.field, self.context_id @@ -155,7 +155,7 @@ class TransactionClause(BaseClause): """ A single variable iff statement """ def __unicode__(self): - return u'"{}" = %({})s'.format(self.field, self.context_id) + return u'"{0}" = %({1})s'.format(self.field, self.context_id) def insert_tuple(self): return self.field, self.context_id @@ -199,9 +199,9 @@ def __unicode__(self): self._assignments is None and self._additions is None and self._removals is None): - qs += ['"{}" = %({})s'.format(self.field, ctx_id)] + qs += ['"{0}" = %({1})s'.format(self.field, ctx_id)] if self._assignments is not None: - qs += ['"{}" = %({})s'.format(self.field, ctx_id)] + qs += ['"{0}" = %({1})s'.format(self.field, ctx_id)] ctx_id += 1 if self._additions is not None: qs += ['"{0}" = "{0}" + %({1})s'.format(self.field, ctx_id)] @@ -270,7 +270,7 @@ def __unicode__(self): qs = [] ctx_id = self.context_id if self._assignments is not None: - qs += ['"{}" = %({})s'.format(self.field, ctx_id)] + qs += ['"{0}" = %({1})s'.format(self.field, ctx_id)] ctx_id += 1 if self._prepend is not None: @@ -398,10 +398,10 @@ def __unicode__(self): ctx_id = self.context_id if self.previous is None and not self._updates: - qs += ['"{}" = %({})s'.format(self.field, ctx_id)] + qs += ['"{0}" = %({1})s'.format(self.field, ctx_id)] else: for _ in self._updates or []: - qs += ['"{}"[%({})s] = %({})s'.format(self.field, ctx_id, ctx_id + 1)] + qs += ['"{0}"[%({1})s] = %({2})s'.format(self.field, ctx_id, ctx_id + 1)] ctx_id += 2 return ', '.join(qs) @@ -436,7 +436,7 @@ def __init__(self, field): super(FieldDeleteClause, self).__init__(field, None) def __unicode__(self): - return '"{}"'.format(self.field) + return '"{0}"'.format(self.field) def update_context(self, ctx): pass @@ -473,7 +473,7 @@ def get_context_size(self): def __unicode__(self): if not self._analyzed: self._analyze() - return ', '.join(['"{}"[%({})s]'.format(self.field, self.context_id + i) for i in range(len(self._removals))]) + return ', '.join(['"{0}"[%({1})s]'.format(self.field, self.context_id + i) for i in range(len(self._removals))]) class BaseCQLStatement(UnicodeMixin): @@ -550,7 +550,7 @@ def __repr__(self): @property def _where(self): - return 'WHERE {}'.format(' AND '.join([six.text_type(c) for c in self.where_clauses])) + return 'WHERE {0}'.format(' AND '.join([six.text_type(c) for c in self.where_clauses])) class SelectStatement(BaseCQLStatement): @@ -587,17 +587,17 @@ def __unicode__(self): if self.count: qs += ['COUNT(*)'] else: - qs += [', '.join(['"{}"'.format(f) for f in self.fields]) if self.fields else '*'] + qs += [', '.join(['"{0}"'.format(f) for f in self.fields]) if self.fields else '*'] qs += ['FROM', self.table] if self.where_clauses: qs += [self._where] if self.order_by and not self.count: - qs += ['ORDER BY {}'.format(', '.join(six.text_type(o) for o in self.order_by))] + qs += ['ORDER BY {0}'.format(', '.join(six.text_type(o) for o in self.order_by))] if self.limit: - qs += ['LIMIT {}'.format(self.limit)] + qs += ['LIMIT {0}'.format(self.limit)] if self.allow_filtering: qs += ['ALLOW FILTERING'] @@ -681,24 +681,24 @@ def add_where_clause(self, clause): raise StatementException("Cannot add where clauses to insert statements") def __unicode__(self): - qs = ['INSERT INTO {}'.format(self.table)] + qs = ['INSERT INTO {0}'.format(self.table)] # get column names and context placeholders fields = [a.insert_tuple() for a in self.assignments] columns, values = zip(*fields) - qs += ["({})".format(', '.join(['"{}"'.format(c) for c in columns]))] + qs += ["({0})".format(', '.join(['"{0}"'.format(c) for c in columns]))] qs += ['VALUES'] - qs += ["({})".format(', '.join(['%({})s'.format(v) for v in values]))] + qs += ["({0})".format(', '.join(['%({0})s'.format(v) for v in values]))] if self.if_not_exists: qs += ["IF NOT EXISTS"] if self.ttl: - qs += ["USING TTL {}".format(self.ttl)] + qs += ["USING TTL {0}".format(self.ttl)] if self.timestamp: - qs += ["USING TIMESTAMP {}".format(self.timestamp_normalized)] + qs += ["USING TIMESTAMP {0}".format(self.timestamp_normalized)] return ' '.join(qs) @@ -732,13 +732,13 @@ def __unicode__(self): using_options = [] if self.ttl: - using_options += ["TTL {}".format(self.ttl)] + using_options += ["TTL {0}".format(self.ttl)] if self.timestamp: - using_options += ["TIMESTAMP {}".format(self.timestamp_normalized)] + using_options += ["TIMESTAMP {0}".format(self.timestamp_normalized)] if using_options: - qs += ["USING {}".format(" AND ".join(using_options))] + qs += ["USING {0}".format(" AND ".join(using_options))] qs += ['SET'] qs += [', '.join([six.text_type(c) for c in self.assignments])] @@ -771,7 +771,7 @@ def get_context(self): return ctx def _get_transactions(self): - return 'IF {}'.format(' AND '.join([six.text_type(c) for c in self.transactions])) + return 'IF {0}'.format(' AND '.join([six.text_type(c) for c in self.transactions])) def update_context_id(self, i): super(UpdateStatement, self).update_context_id(i) @@ -820,16 +820,16 @@ def add_field(self, field): def __unicode__(self): qs = ['DELETE'] if self.fields: - qs += [', '.join(['{}'.format(f) for f in self.fields])] + qs += [', '.join(['{0}'.format(f) for f in self.fields])] qs += ['FROM', self.table] delete_option = [] if self.timestamp: - delete_option += ["TIMESTAMP {}".format(self.timestamp_normalized)] + delete_option += ["TIMESTAMP {0}".format(self.timestamp_normalized)] if delete_option: - qs += [" USING {} ".format(" AND ".join(delete_option))] + qs += [" USING {0} ".format(" AND ".join(delete_option))] if self.where_clauses: qs += [self._where] diff --git a/cassandra/cqlengine/usertype.py b/cassandra/cqlengine/usertype.py index 1e30fc829c..88ec033ba8 100644 --- a/cassandra/cqlengine/usertype.py +++ b/cassandra/cqlengine/usertype.py @@ -56,7 +56,7 @@ def __ne__(self, other): return not self.__eq__(other) def __str__(self): - return "{{{}}}".format(', '.join("'{}': {}".format(k, getattr(self, k)) for k, v in six.iteritems(self._values))) + return "{{{0}}}".format(', '.join("'{0}': {1}".format(k, getattr(self, k)) for k, v in six.iteritems(self._values))) def has_changed_fields(self): return any(v.changed for v in self._values.values()) @@ -116,7 +116,7 @@ def type_name(cls): type_name = cls.__type_name__.lower() else: camelcase = re.compile(r'([a-z])([A-Z])') - ccase = lambda s: camelcase.sub(lambda v: '{}_{}'.format(v.group(1), v.group(2)), s) + ccase = lambda s: camelcase.sub(lambda v: '{0}_{1}'.format(v.group(1), v.group(2)), s) type_name = ccase(cls.__name__) # trim to less than 48 characters or cassandra will complain @@ -157,7 +157,7 @@ def _transform_column(field_name, field_obj): for k, v in field_defs: # don't allow a field with the same name as a built-in attribute or method if k in BaseUserType.__dict__: - raise UserTypeDefinitionException("field '{}' conflicts with built-in attribute/method".format(k)) + raise UserTypeDefinitionException("field '{0}' conflicts with built-in attribute/method".format(k)) _transform_column(k, v) # create db_name -> model name map for loading From 8bfcfb8824ce46f0d1cfc2735f70d916cabceb81 Mon Sep 17 00:00:00 2001 From: jpuerta Date: Mon, 1 Jun 2015 11:33:43 +0100 Subject: [PATCH 0181/2431] cqle: Fix minor issues added in previous commit Changes: - Extra whitespace added during merge - Replace (k, v) syntax in dict comprehension with k, v - Discriminator column for polymorphic key --- cassandra/cqlengine/columns.py | 2 +- cassandra/cqlengine/models.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index b5c49ecea6..bba3232c6f 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -563,7 +563,7 @@ def from_datetime(self, dt): epoch = datetime(1970, 1, 1, tzinfo=dt.tzinfo) offset = get_total_seconds(epoch.tzinfo.utcoffset(epoch)) if epoch.tzinfo else 0 - timestamp = get_total_seconds(dt - epoch) - offset + timestamp = get_total_seconds(dt - epoch) - offset node = None clock_seq = None diff --git a/cassandra/cqlengine/models.py b/cassandra/cqlengine/models.py index b0f86d990e..58fc8a94e3 100644 --- a/cassandra/cqlengine/models.py +++ b/cassandra/cqlengine/models.py @@ -434,7 +434,7 @@ def _construct_instance(cls, values): klass = poly_base._get_model_by_discriminator_value(disc_key) if klass is None: raise PolyMorphicModelException( - 'unrecognized polymorphic key {0} for class {1}'.format(poly_key, poly_base.__name__) + 'unrecognized discriminator column {0} for class {1}'.format(poly_key, poly_base.__name__) ) if not issubclass(klass, cls): @@ -442,7 +442,7 @@ def _construct_instance(cls, values): '{0} is not a subclass of {1}'.format(klass.__name__, cls.__name__) ) - field_dict = dict((k, v) for (k, v) in field_dict.items() if k in klass._columns.keys()) + field_dict = dict((k, v) for k, v in field_dict.items() if k in klass._columns.keys()) else: klass = cls From be0d0037ea0227897e665be1f522ab66e20ad084 Mon Sep 17 00:00:00 2001 From: jpuerta Date: Tue, 2 Jun 2015 16:29:12 +0100 Subject: [PATCH 0182/2431] cqle: Fix minor issues with polymorphic models --- cassandra/cqlengine/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cassandra/cqlengine/models.py b/cassandra/cqlengine/models.py index 58fc8a94e3..ddb7945995 100644 --- a/cassandra/cqlengine/models.py +++ b/cassandra/cqlengine/models.py @@ -433,12 +433,12 @@ def _construct_instance(cls, values): poly_base._discover_polymorphic_submodels() klass = poly_base._get_model_by_discriminator_value(disc_key) if klass is None: - raise PolyMorphicModelException( - 'unrecognized discriminator column {0} for class {1}'.format(poly_key, poly_base.__name__) + raise PolymorphicModelException( + 'unrecognized discriminator column {0} for class {1}'.format(disc_key, poly_base.__name__) ) if not issubclass(klass, cls): - raise PolyMorphicModelException( + raise PolymorphicModelException( '{0} is not a subclass of {1}'.format(klass.__name__, cls.__name__) ) From 8949200222343407ba6201a59f10169f089be2de Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 2 Jun 2015 12:11:32 -0500 Subject: [PATCH 0183/2431] Update custom payload value encoding for CASSADNRA-9515 https://issues.apache.org/jira/browse/CASSANDRA-9515 --- cassandra/protocol.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 1c851f8c5d..20904ecec3 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -1019,11 +1019,6 @@ def read_binary_string(f): return contents -def write_binary_string(f, s): - write_short(f, len(s)) - f.write(s) - - def write_string(f, s): if isinstance(s, six.text_type): s = s.encode('utf8') @@ -1080,7 +1075,7 @@ def read_bytesmap(f): bytesmap = {} for _ in range(numpairs): k = read_string(f) - bytesmap[k] = read_binary_string(f) + bytesmap[k] = read_value(f) return bytesmap @@ -1088,7 +1083,7 @@ def write_bytesmap(f, bytesmap): write_short(f, len(bytesmap)) for k, v in bytesmap.items(): write_string(f, k) - write_binary_string(f, v) + write_value(f, v) def read_stringmultimap(f): From ec241de0feb2c888a59b7c3a935f0562d8084716 Mon Sep 17 00:00:00 2001 From: Ashwin Rajeev Date: Thu, 4 Jun 2015 12:52:38 +0530 Subject: [PATCH 0184/2431] fix unicode encoding issue for query.bind_params --- cassandra/cluster.py | 2 -- cassandra/query.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cb2502e7d2..2708736089 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1653,8 +1653,6 @@ def _create_response_future(self, query, parameters, trace, custom_payload): if isinstance(query, SimpleStatement): query_string = query.query_string - if six.PY2 and isinstance(query_string, six.text_type): - query_string = query_string.encode('utf-8') if parameters: query_string = bind_params(query_string, parameters, self.encoder) message = QueryMessage( diff --git a/cassandra/query.py b/cassandra/query.py index 6709cbeff6..19d227beb8 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -793,6 +793,8 @@ def __str__(self): def bind_params(query, params, encoder): + if six.PY2 and isinstance(query, six.text_type): + query = query.encode('utf-8') if isinstance(params, dict): return query % dict((k, encoder.cql_encode_all_types(v)) for k, v in six.iteritems(params)) else: From cf1432683265bc2e232ec8f060171be7ef0bc1b3 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 4 Jun 2015 09:06:58 -0500 Subject: [PATCH 0185/2431] Finalize changelog for 2.6.0c1 --- CHANGELOG.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 63f4185c30..0ba2a4ac6c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,6 @@ 2.6.0rc1 ===== +June 4, 2015 This release adds support for Cassandra 2.2 features, including version 4 of the native protocol. @@ -15,7 +16,7 @@ Features * Support new types in C* 2.2: date, time, smallint, tinyint (PYTHON-245, 295) * cqle: add Double column type and remove Float overload (PYTHON-246) * Use partition key column information in prepared response for protocol v4+ (PYTHON-277) -* Support message custom payloads in protocol v4+ (PYTHON-280) +* Support message custom payloads in protocol v4+ (PYTHON-280, PYTHON-329) * Deprecate refresh_schema and replace with functions for specific entities (PYTHON-291) * Save trace id even when trace complete times out (PYTHON-302) * Warn when registering client UDT class for protocol < v3 (PYTHON-305) From 0dc0d584de0a8201ea249572cbf33a6b589e7fdc Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 4 Jun 2015 09:08:33 -0500 Subject: [PATCH 0186/2431] 2.6.0c1 version --- cassandra/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 3f61909664..379f6d4534 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 5, 1, 'post') +__version_info__ = (2, 6, '0c1') __version__ = '.'.join(map(str, __version_info__)) From db9bf9b7973441282217a03001dbbcb8b6c0d8a5 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 4 Jun 2015 09:57:51 -0500 Subject: [PATCH 0187/2431] Fix cassandra.protocol API docs --- docs/api/cassandra/protocol.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/api/cassandra/protocol.rst b/docs/api/cassandra/protocol.rst index a6b4d1c41a..52f1287a6c 100644 --- a/docs/api/cassandra/protocol.rst +++ b/docs/api/cassandra/protocol.rst @@ -1,7 +1,12 @@ +``cassandra.protocol`` - Protocol Features +===================================================================== + +.. module:: cassandra.protocol + .. _custom_payload: -Custom Payload -============== +Custom Payloads +--------------- Native protocol version 4+ allows for a custom payload to be sent between clients and custom query handlers. The payload is specified as a string:binary_type dict holding custom key/value pairs. @@ -9,3 +14,4 @@ holding custom key/value pairs. By default these are ignored by the server. They can be useful for servers implementing a custom QueryHandler. +See :meth:`.Session.execute`, ::meth:`.Session.execute_async`, :attr:`.ResponseFuture.custom_payload`. From 48ec3ebee2bfecefb1689f1864da063dd115128c Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 4 Jun 2015 11:35:43 -0500 Subject: [PATCH 0188/2431] Correct release version in changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0ba2a4ac6c..41b79b7e54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,4 +1,4 @@ -2.6.0rc1 +2.6.0c1 ===== June 4, 2015 From f5c8aa952255c6196c6acfeec448889915305172 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 4 Jun 2015 11:59:13 -0500 Subject: [PATCH 0189/2431] Post release version --- cassandra/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 379f6d4534..059aeb00eb 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 6, '0c1') +__version_info__ = (2, 6, '0c1', 'post') __version__ = '.'.join(map(str, __version_info__)) From be0333626bf1c20a0ea85f359035d69b59efca2d Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 4 Jun 2015 13:04:17 -0500 Subject: [PATCH 0190/2431] Add custom_payload to Session.execute doc annotation --- docs/api/cassandra/cluster.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 195d03a6ef..3edc22c82a 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -106,9 +106,9 @@ .. autoattribute:: encoder - .. automethod:: execute(statement[, parameters][, timeout][, trace]) + .. automethod:: execute(statement[, parameters][, timeout][, trace][, custom_payload]) - .. automethod:: execute_async(statement[, parameters][, trace]) + .. automethod:: execute_async(statement[, parameters][, trace][, custom_payload]) .. automethod:: prepare(statement) From 088a546e5281096a9628b07ec2a5602793bc5db2 Mon Sep 17 00:00:00 2001 From: Ricardo Sancho Date: Sat, 6 Jun 2015 11:41:31 +0100 Subject: [PATCH 0191/2431] Write timeouts being ignored with 0 responses DowngradingConsistencyRetryPolicy was ignoring write timeouts for WriteTypes SIMPLE, BATCH and COUNTER even if responses received was equal to 0. It is valid to ignore the failure only if at least one response was received. --- cassandra/policies.py | 2 +- tests/unit/test_policies.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index d4d8914c8b..6d6c23b972 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -816,7 +816,7 @@ def on_write_timeout(self, query, consistency, write_type, required_responses, received_responses, retry_num): if retry_num != 0: return (self.RETHROW, None) - elif write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER): + elif write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER) and received_responses > 0: return (self.IGNORE, None) elif write_type == WriteType.UNLOGGED_BATCH: return self._pick_consistency(received_responses) diff --git a/tests/unit/test_policies.py b/tests/unit/test_policies.py index 4f865fb1ee..bd8541047f 100644 --- a/tests/unit/test_policies.py +++ b/tests/unit/test_policies.py @@ -1052,6 +1052,14 @@ def test_write_timeout(self): self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) + # On these type of writes failures should not be ignored + # if received_responses is 0 + for write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER): + retry, consistency = policy.on_write_timeout( + query=None, consistency=ONE, write_type=write_type, + required_responses=1, received_responses=0, retry_num=0) + self.assertEqual(retry, RetryPolicy.RETHROW) + # ignore failures on these types of writes for write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER): retry, consistency = policy.on_write_timeout( From b48944ec61a83b4a914da803c187a526cf8a7982 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 8 Jun 2015 08:54:09 -0500 Subject: [PATCH 0192/2431] Remove warning on missing blist Warning is not required, especially since benchmarks show the pure Python impl is faster for most core use cases. --- cassandra/util.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cassandra/util.py b/cassandra/util.py index 45ef0e4fe7..ba49b5687e 100644 --- a/cassandra/util.py +++ b/cassandra/util.py @@ -480,13 +480,6 @@ def isdisjoint(self, other): from blist import sortedset except ImportError: - import warnings - - warnings.warn( - "The blist library is not available, so a pure python list-based set will " - "be used in place of blist.sortedset for set collection values. " - "You can find the blist library here: https://pypi.python.org/pypi/blist/") - from bisect import bisect_left class sortedset(object): From 015a0e18efef8a06d218028c189d373dcf52b3c1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 8 Jun 2015 17:45:26 -0500 Subject: [PATCH 0193/2431] Automatically downgrading protocol version on control connection PYTHON-240 --- cassandra/cluster.py | 39 +++++-- cassandra/connection.py | 146 +++++++++++++------------- cassandra/io/asyncorereactor.py | 1 - cassandra/io/eventletreactor.py | 1 - cassandra/io/geventreactor.py | 1 - cassandra/io/libevreactor.py | 1 - cassandra/io/twistedreactor.py | 1 - cassandra/protocol.py | 6 +- tests/unit/io/test_asyncorereactor.py | 23 ++-- tests/unit/io/test_libevreactor.py | 25 +++-- tests/unit/io/test_twistedreactor.py | 18 ++-- tests/unit/test_connection.py | 81 ++++++-------- 12 files changed, 178 insertions(+), 165 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cb2502e7d2..bd3eefeea7 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -47,7 +47,7 @@ InvalidRequest, OperationTimedOut, UnsupportedOperation, Unauthorized) from cassandra.connection import (ConnectionException, ConnectionShutdown, - ConnectionHeartbeat) + ConnectionHeartbeat, ProtocolVersionUnsupported) from cassandra.cqltypes import UserType from cassandra.encoder import Encoder from cassandra.protocol import (QueryMessage, ResultMessage, @@ -60,7 +60,7 @@ IsBootstrappingErrorMessage, BatchMessage, RESULT_KIND_PREPARED, RESULT_KIND_SET_KEYSPACE, RESULT_KIND_ROWS, - RESULT_KIND_SCHEMA_CHANGE) + RESULT_KIND_SCHEMA_CHANGE, MIN_SUPPORTED_VERSION) from cassandra.metadata import Metadata, protect_name from cassandra.policies import (TokenAwarePolicy, DCAwareRoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, @@ -220,9 +220,14 @@ class Cluster(object): server will be automatically used. """ - protocol_version = 2 + protocol_version = 4 """ - The version of the native protocol to use. + The maximum version of the native protocol to use. + + The driver will automatically downgrade version based on a negotiation with + the server, but it is most efficient to set this to the maximum supported + by your version of Cassandra. Setting this will also prevent conflicting + versions negotiated if your cluster is upgraded. Version 2 of the native protocol adds support for lightweight transactions, batch operations, and automatic query paging. The v2 protocol is @@ -233,6 +238,10 @@ class Cluster(object): serial consistency levels for :class:`~.BatchStatement`, and an improved connection pool. + Version 4 of the native protocol adds a number of new types, server warnings, + new failure messages, and custom payloads. Details in the + `project docs `_ + The following table describes the native protocol versions that are supported by each version of Cassandra: @@ -245,6 +254,8 @@ class Cluster(object): +-------------------+-------------------+ | 2.1 | 1, 2, 3 | +-------------------+-------------------+ + | 2.2 | 1, 2, 3, 4 | + +-------------------+-------------------+ """ compression = True @@ -495,7 +506,7 @@ def __init__(self, ssl_options=None, sockopts=None, cql_version=None, - protocol_version=2, + protocol_version=4, executor_threads=2, max_schema_agreement_wait=10, control_connection_timeout=2.0, @@ -788,6 +799,15 @@ def _make_connection_kwargs(self, address, kwargs_dict): return kwargs_dict + def protocol_downgrade(self, host_addr, previous_version): + new_version = previous_version - 1 + if new_version < self.protocol_version: + if new_version >= MIN_SUPPORTED_VERSION: + log.warning("Downgrading core protocol version from %d to %d for %s", self.protocol_version, new_version, host_addr) + self.protocol_version = new_version + else: + raise Exception("Cannot downgrade protocol version (%d) below minimum supported version: %d" % (new_version, MIN_SUPPORTED_VERSION)) + def connect(self, keyspace=None): """ Creates and returns a new :class:`~.Session` object. If `keyspace` @@ -1503,7 +1523,6 @@ class Session(object): _pools = None _load_balancer = None _metrics = None - _protocol_version = None def __init__(self, cluster, hosts): self.cluster = cluster @@ -2094,7 +2113,13 @@ def _try_connect(self, host): node/token and schema metadata. """ log.debug("[control connection] Opening new connection to %s", host) - connection = self._cluster.connection_factory(host.address, is_control_connection=True) + + while True: + try: + connection = self._cluster.connection_factory(host.address, is_control_connection=True) + break + except ProtocolVersionUnsupported as e: + self._cluster.protocol_downgrade(host.address, e.startup_version) log.debug("[control connection] Established new connection %r, " "registering watchers and refreshing schema and topology", diff --git a/cassandra/connection.py b/cassandra/connection.py index c239313c45..bf0e5f0f69 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -13,12 +13,12 @@ # limitations under the License. from __future__ import absolute_import # to enable import io from stdlib -from collections import defaultdict, deque +from collections import defaultdict, deque, namedtuple import errno from functools import wraps, partial import io import logging -import os +import struct import sys from threading import Thread, Event, RLock import time @@ -32,13 +32,13 @@ from six.moves import range from cassandra import ConsistencyLevel, AuthenticationFailed, OperationTimedOut -from cassandra.marshal import int32_pack, header_unpack, v3_header_unpack, int32_unpack +from cassandra.marshal import int32_pack, uint8_unpack from cassandra.protocol import (ReadyMessage, AuthenticateMessage, OptionsMessage, StartupMessage, ErrorMessage, CredentialsMessage, QueryMessage, ResultMessage, decode_response, InvalidRequestException, SupportedMessage, AuthResponseMessage, AuthChallengeMessage, - AuthSuccessMessage, ProtocolException) + AuthSuccessMessage, ProtocolException, MAX_SUPPORTED_VERSION) from cassandra.util import OrderedDict @@ -88,6 +88,11 @@ def decompress(byts): HEADER_DIRECTION_TO_CLIENT = 0x80 HEADER_DIRECTION_MASK = 0x80 +frame_header_v1_v2 = struct.Struct('>BbBi') +frame_header_v3 = struct.Struct('>BhBi') + +_Frame = namedtuple('Frame', ('version', 'flags', 'stream', 'opcode', 'body_offset', 'end_pos')) + NONBLOCKING = (errno.EAGAIN, errno.EWOULDBLOCK) @@ -108,6 +113,14 @@ class ConnectionShutdown(ConnectionException): """ pass +class ProtocolVersionUnsupported(ConnectionException): + """ + Server rejected startup message due to unsupported protocol version + """ + def __init__(self, host, startup_version): + super(ProtocolVersionUnsupported, self).__init__("Unsupported protocol version on %s: %d", + (host, startup_version)) + self.startup_version = startup_version class ConnectionBusy(Exception): """ @@ -144,7 +157,7 @@ class Connection(object): out_buffer_size = 4096 cql_version = None - protocol_version = 2 + protocol_version = MAX_SUPPORTED_VERSION keyspace = None compression = True @@ -174,12 +187,15 @@ class Connection(object): msg_received = False + is_unsupported_proto_version = False + is_control_connection = False _iobuf = None + _current_frame = None def __init__(self, host='127.0.0.1', port=9042, authenticator=None, ssl_options=None, sockopts=None, compression=True, - cql_version=None, protocol_version=2, is_control_connection=False, + cql_version=None, protocol_version=MAX_SUPPORTED_VERSION, is_control_connection=False, user_type_map=None): self.host = host self.port = port @@ -193,34 +209,18 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.user_type_map = user_type_map self._push_watchers = defaultdict(set) self._iobuf = io.BytesIO() + if protocol_version >= 3: - self._header_unpack = v3_header_unpack - self._header_length = 5 self.max_request_id = (2 ** 15) - 1 # Don't fill the deque with 2**15 items right away. Start with 300 and add # more if needed. self.request_ids = deque(range(300)) self.highest_request_id = 299 else: - self._header_unpack = header_unpack - self._header_length = 4 self.max_request_id = (2 ** 7) - 1 self.request_ids = deque(range(self.max_request_id + 1)) self.highest_request_id = self.max_request_id - # 0 8 16 24 32 40 - # +---------+---------+---------+---------+---------+ - # | version | flags | stream | opcode | - # +---------+---------+---------+---------+---------+ - # | length | - # +---------+---------+---------+---------+ - # | | - # . ... body ... . - # . . - # . . - # +---------------------------------------- - self._full_header_length = self._header_length + 4 - self.lock = RLock() @classmethod @@ -249,6 +249,8 @@ def factory(cls, host, timeout, *args, **kwargs): conn = cls(host, *args, **kwargs) conn.connected_event.wait(timeout) if conn.last_error: + if conn.is_unsupported_proto_version: + raise ProtocolVersionUnsupported(host, conn.protocol_version) raise conn.last_error elif not conn.connected_event.is_set(): conn.close() @@ -377,42 +379,56 @@ def control_conn_disposed(self): self.is_control_connection = False self._push_watchers = {} + @defunct_on_error + def _read_frame_header(self): + buf = self._iobuf + pos = buf.tell() + if pos: + buf.seek(0) + version = uint8_unpack(buf.read(1)) & PROTOCOL_VERSION_MASK + if version > MAX_SUPPORTED_VERSION: + raise ProtocolError("This version of the driver does not support protocol version %d" % version) + frame_header = frame_header_v3 if version >= 3 else frame_header_v1_v2 + # this frame header struct is everything after the version byte + header_size = frame_header.size + 1 + if pos >= header_size: + flags, stream, op, body_len = frame_header.unpack(buf.read(frame_header.size)) + if body_len < 0: + raise ProtocolError("Received negative body length: %r" % body_len) + self._current_frame = _Frame(version, flags, stream, op, header_size, body_len + header_size) + + self._iobuf.seek(pos) + + return pos + + def _reset_frame(self): + leftover = self._iobuf.read() + self._iobuf = io.BytesIO() + self._iobuf.write(leftover) + self._current_frame = None + def process_io_buffer(self): while True: - pos = self._iobuf.tell() - if pos < self._full_header_length or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes): + if not self._current_frame: + pos = self._read_frame_header() + else: + pos = self._iobuf.tell() + + if not self._current_frame or pos < self._current_frame.end_pos: # we don't have a complete header yet or we # already saw a header, but we don't have a # complete message yet return else: - # have enough for header, read body len from header - self._iobuf.seek(self._header_length) - body_len = int32_unpack(self._iobuf.read(4)) - - # seek to end to get length of current buffer - self._iobuf.seek(0, os.SEEK_END) - pos = self._iobuf.tell() - - if pos >= body_len + self._full_header_length: - # read message header and body - self._iobuf.seek(0) - msg = self._iobuf.read(self._full_header_length + body_len) - - # leave leftover in current buffer - leftover = self._iobuf.read() - self._iobuf = io.BytesIO() - self._iobuf.write(leftover) - - self._total_reqd_bytes = 0 - self.process_msg(msg, body_len) - else: - self._total_reqd_bytes = body_len + self._full_header_length - return + frame = self._current_frame + self._iobuf.seek(frame.body_offset) + msg = self._iobuf.read(frame.end_pos - frame.body_offset) + self.process_msg(self._current_frame, msg) + self._reset_frame() @defunct_on_error - def process_msg(self, msg, body_len): - version, flags, stream_id, opcode = self._header_unpack(msg[:self._header_length]) + def process_msg(self, header, body): + stream_id = header.stream if stream_id < 0: callback = None else: @@ -422,33 +438,12 @@ def process_msg(self, msg, body_len): self.msg_received = True - body = None try: - # check that the protocol version is supported - given_version = version & PROTOCOL_VERSION_MASK - if given_version != self.protocol_version: - msg = "Server protocol version (%d) does not match the specified driver protocol version (%d). " +\ - "Consider setting Cluster.protocol_version to %d." - raise ProtocolError(msg % (given_version, self.protocol_version, given_version)) - - # check that the header direction is correct - if version & HEADER_DIRECTION_MASK != HEADER_DIRECTION_TO_CLIENT: - raise ProtocolError( - "Header direction in response is incorrect; opcode %04x, stream id %r" - % (opcode, stream_id)) - - if body_len > 0: - body = msg[self._full_header_length:] - elif body_len == 0: - body = six.binary_type() - else: - raise ProtocolError("Got negative body length: %r" % body_len) - - response = decode_response(given_version, self.user_type_map, stream_id, - flags, opcode, body, self.decompressor) + response = decode_response(header.version, self.user_type_map, stream_id, + header.flags, header.opcode, body, self.decompressor) except Exception as exc: log.exception("Error decoding response from Cassandra. " - "opcode: %04x; message contents: %r", opcode, msg) + "opcode: %04x; message contents: %r", header.opcode, body) if callback is not None: callback(exc) self.defunct(exc) @@ -457,6 +452,9 @@ def process_msg(self, msg, body_len): try: if stream_id >= 0: if isinstance(response, ProtocolException): + if 'unsupported protocol version' in response.message: + self.is_unsupported_proto_version = True + log.error("Closing connection %s due to protocol error: %s", self, response.summary_msg()) self.defunct(response) if callback is not None: diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index ef687c388c..e5db1c74cc 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -135,7 +135,6 @@ class AsyncoreConnection(Connection, asyncore.dispatcher): _loop = None - _total_reqd_bytes = 0 _writable = False _readable = False diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 670d0f1865..f123e0bd68 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -48,7 +48,6 @@ class EventletConnection(Connection): An implementation of :class:`.Connection` that utilizes ``eventlet``. """ - _total_reqd_bytes = 0 _read_watcher = None _write_watcher = None _socket = None diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 6e9af0da4d..cdf1b25451 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -45,7 +45,6 @@ class GeventConnection(Connection): An implementation of :class:`.Connection` that utilizes ``gevent``. """ - _total_reqd_bytes = 0 _read_watcher = None _write_watcher = None _socket = None diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 93b4c97854..ca27bee800 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -215,7 +215,6 @@ class LibevConnection(Connection): """ _libevloop = None _write_watcher_is_active = False - _total_reqd_bytes = 0 _read_watcher = None _write_watcher = None _socket = None diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index ff81e5613f..425c96ce00 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -141,7 +141,6 @@ class TwistedConnection(Connection): """ _loop = None - _total_reqd_bytes = 0 @classmethod def initialize_reactor(cls): diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 20904ecec3..9b544236ec 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -53,7 +53,9 @@ class InternalError(Exception): ColumnMetadata = namedtuple("ColumnMetadata", ['keyspace_name', 'table_name', 'name', 'type']) -HEADER_DIRECTION_FROM_CLIENT = 0x00 +MIN_SUPPORTED_VERSION = 1 +MAX_SUPPORTED_VERSION = 4 + HEADER_DIRECTION_TO_CLIENT = 0x80 HEADER_DIRECTION_MASK = 0x80 @@ -967,7 +969,7 @@ def write_header(f, version, flags, stream_id, opcode, length): Write a CQL protocol frame header. """ pack = v3_header_pack if version >= 3 else header_pack - f.write(pack(version | HEADER_DIRECTION_FROM_CLIENT, flags, stream_id, opcode)) + f.write(pack(version, flags, stream_id, opcode)) write_int(f, length) diff --git a/tests/unit/io/test_asyncorereactor.py b/tests/unit/io/test_asyncorereactor.py index 17e42a0221..cdc61ec2d2 100644 --- a/tests/unit/io/test_asyncorereactor.py +++ b/tests/unit/io/test_asyncorereactor.py @@ -20,17 +20,15 @@ import unittest # noqa import errno +import math +from mock import patch, Mock import os - from six import BytesIO - import socket from socket import error as socket_error -from mock import patch, Mock - from cassandra.connection import (HEADER_DIRECTION_TO_CLIENT, - ConnectionException) + ConnectionException, ProtocolError) from cassandra.io.asyncorereactor import AsyncoreConnection from cassandra.protocol import (write_stringmultimap, write_int, write_string, SupportedMessage, ReadyMessage, ServerError) @@ -132,7 +130,7 @@ def side_effect(*args): c.socket.recv.side_effect = side_effect c.handle_read() - self.assertEqual(c._total_reqd_bytes, 20000 + len(header)) + self.assertEqual(c._current_frame.end_pos, 20000 + len(header)) # the EAGAIN prevents it from reading the last 100 bytes c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() @@ -159,7 +157,7 @@ def test_protocol_error(self, *args): # make sure it errored correctly self.assertTrue(c.is_defunct) self.assertTrue(c.connected_event.is_set()) - self.assertIsInstance(c.last_error, ConnectionException) + self.assertIsInstance(c.last_error, ProtocolError) def test_error_message_on_startup(self, *args): c = self.make_connection() @@ -217,13 +215,18 @@ def test_partial_send(self, *args): c = self.make_connection() # only write the first four bytes of the OptionsMessage + write_size = 4 c.socket.send.side_effect = None - c.socket.send.return_value = 4 + c.socket.send.return_value = write_size c.handle_write() + msg_size = 9 # v3+ frame header + expected_writes = int(math.ceil(float(msg_size) / write_size)) + size_mod = msg_size % write_size + last_write_size = size_mod if size_mod else write_size self.assertFalse(c.is_defunct) - self.assertEqual(2, c.socket.send.call_count) - self.assertEqual(4, len(c.socket.send.call_args[0][0])) + self.assertEqual(expected_writes, c.socket.send.call_count) + self.assertEqual(last_write_size, len(c.socket.send.call_args[0][0])) def test_socket_error_on_read(self, *args): c = self.make_connection() diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index ddd2dc0417..61e72421ce 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -17,18 +17,16 @@ import unittest # noqa import errno +import math +from mock import patch, Mock import os -import sys - import six from six import BytesIO - from socket import error as socket_error - -from mock import patch, Mock +import sys from cassandra.connection import (HEADER_DIRECTION_TO_CLIENT, - ConnectionException) + ConnectionException, ProtocolError) from cassandra.protocol import (write_stringmultimap, write_int, write_string, SupportedMessage, ReadyMessage, ServerError) @@ -128,7 +126,7 @@ def side_effect(*args): c._socket.recv.side_effect = side_effect c.handle_read(None, 0) - self.assertEqual(c._total_reqd_bytes, 20000 + len(header)) + self.assertEqual(c._current_frame.end_pos, 20000 + len(header)) # the EAGAIN prevents it from reading the last 100 bytes c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() @@ -155,7 +153,7 @@ def test_protocol_error(self, *args): # make sure it errored correctly self.assertTrue(c.is_defunct) self.assertTrue(c.connected_event.is_set()) - self.assertIsInstance(c.last_error, ConnectionException) + self.assertIsInstance(c.last_error, ProtocolError) def test_error_message_on_startup(self, *args): c = self.make_connection() @@ -213,13 +211,18 @@ def test_partial_send(self, *args): c = self.make_connection() # only write the first four bytes of the OptionsMessage + write_size = 4 c._socket.send.side_effect = None - c._socket.send.return_value = 4 + c._socket.send.return_value = write_size c.handle_write(None, 0) + msg_size = 9 # v3+ frame header + expected_writes = int(math.ceil(float(msg_size) / write_size)) + size_mod = msg_size % write_size + last_write_size = size_mod if size_mod else write_size self.assertFalse(c.is_defunct) - self.assertEqual(2, c._socket.send.call_count) - self.assertEqual(4, len(c._socket.send.call_args[0][0])) + self.assertEqual(expected_writes, c._socket.send.call_count) + self.assertEqual(last_write_size, len(c._socket.send.call_args[0][0])) def test_socket_error_on_read(self, *args): c = self.make_connection() diff --git a/tests/unit/io/test_twistedreactor.py b/tests/unit/io/test_twistedreactor.py index 8563b948c0..d2142b09ca 100644 --- a/tests/unit/io/test_twistedreactor.py +++ b/tests/unit/io/test_twistedreactor.py @@ -25,6 +25,7 @@ except ImportError: twistedreactor = None # NOQA +from cassandra.connection import _Frame class TestTwistedProtocol(unittest.TestCase): @@ -154,16 +155,16 @@ def test_handle_read__incomplete(self): self.obj_ut.process_msg = Mock() self.assertEqual(self.obj_ut._iobuf.getvalue(), '') # buf starts empty # incomplete header - self.obj_ut._iobuf.write('\xff\x00\x00\x00') + self.obj_ut._iobuf.write('\x84\x00\x00\x00\x00') self.obj_ut.handle_read() - self.assertEqual(self.obj_ut._iobuf.getvalue(), '\xff\x00\x00\x00') + self.assertEqual(self.obj_ut._iobuf.getvalue(), '\x84\x00\x00\x00\x00') # full header, but incomplete body self.obj_ut._iobuf.write('\x00\x00\x00\x15') self.obj_ut.handle_read() self.assertEqual(self.obj_ut._iobuf.getvalue(), - '\xff\x00\x00\x00\x00\x00\x00\x15') - self.assertEqual(self.obj_ut._total_reqd_bytes, 29) + '\x84\x00\x00\x00\x00\x00\x00\x00\x15') + self.assertEqual(self.obj_ut._current_frame.end_pos, 30) # verify we never attempted to process the incomplete message self.assertFalse(self.obj_ut.process_msg.called) @@ -176,12 +177,15 @@ def test_handle_read__fullmessage(self): self.assertEqual(self.obj_ut._iobuf.getvalue(), '') # buf starts empty # write a complete message, plus 'NEXT' (to simulate next message) + # assumes protocol v3+ as default Connection.protocol_version + body = 'this is the drum roll' + extra = 'NEXT' self.obj_ut._iobuf.write( - '\xff\x00\x00\x00\x00\x00\x00\x15this is the drum rollNEXT') + '\x84\x01\x00\x02\x03\x00\x00\x00\x15' + body + extra) self.obj_ut.handle_read() - self.assertEqual(self.obj_ut._iobuf.getvalue(), 'NEXT') + self.assertEqual(self.obj_ut._iobuf.getvalue(), extra) self.obj_ut.process_msg.assert_called_with( - '\xff\x00\x00\x00\x00\x00\x00\x15this is the drum roll', 21) + _Frame(version=4, flags=1, stream=2, opcode=3, body_offset=9, end_pos= 9 + len(body)), body) @patch('twisted.internet.reactor.connectTCP') def test_push(self, mock_connectTCP): diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 7fc4ed4e5a..e5be60759d 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -23,31 +23,38 @@ from threading import Lock from cassandra.cluster import Cluster -from cassandra.connection import (Connection, HEADER_DIRECTION_TO_CLIENT, - HEADER_DIRECTION_FROM_CLIENT, ProtocolError, - locally_supported_compressions, ConnectionHeartbeat) -from cassandra.marshal import uint8_pack, uint32_pack +from cassandra.connection import (Connection, HEADER_DIRECTION_TO_CLIENT, ProtocolError, + locally_supported_compressions, ConnectionHeartbeat, _Frame) +from cassandra.marshal import uint8_pack, uint32_pack, int32_pack from cassandra.protocol import (write_stringmultimap, write_int, write_string, SupportedMessage) class ConnectionTest(unittest.TestCase): - protocol_version = 2 - def make_connection(self): c = Connection('1.2.3.4') c._socket = Mock() c._socket.send.side_effect = lambda x: len(x) return c - def make_header_prefix(self, message_class, version=2, stream_id=0): - return six.binary_type().join(map(uint8_pack, [ - 0xff & (HEADER_DIRECTION_TO_CLIENT | version), - 0, # flags (compression) - stream_id, - message_class.opcode # opcode - ])) + def make_header_prefix(self, message_class, version=Connection.protocol_version, stream_id=0): + if Connection.protocol_version < 3: + return six.binary_type().join(map(uint8_pack, [ + 0xff & (HEADER_DIRECTION_TO_CLIENT | version), + 0, # flags (compression) + stream_id, + message_class.opcode # opcode + ])) + else: + return six.binary_type().join(map(uint8_pack, [ + 0xff & (HEADER_DIRECTION_TO_CLIENT | version), + 0, # flags (compression) + 0, # MSB for v3+ stream + stream_id, + message_class.opcode # opcode + ])) + def make_options_body(self): options_buf = BytesIO() @@ -72,31 +79,12 @@ def test_bad_protocol_version(self, *args): c.defunct = Mock() # read in a SupportedMessage response - header = self.make_header_prefix(SupportedMessage, version=0x04) + header = self.make_header_prefix(SupportedMessage, version=0x7f) options = self.make_options_body() message = self.make_msg(header, options) - c.process_msg(message, len(message) - 8) - - # make sure it errored correctly - c.defunct.assert_called_once_with(ANY) - args, kwargs = c.defunct.call_args - self.assertIsInstance(args[0], ProtocolError) - - def test_bad_header_direction(self, *args): - c = self.make_connection() - c._callbacks = Mock() - c.defunct = Mock() - - # read in a SupportedMessage response - header = six.binary_type().join(uint8_pack(i) for i in ( - 0xff & (HEADER_DIRECTION_FROM_CLIENT | self.protocol_version), - 0, # flags (compression) - 0, - SupportedMessage.opcode # opcode - )) - options = self.make_options_body() - message = self.make_msg(header, options) - c.process_msg(message, len(message) - 8) + c._iobuf = BytesIO() + c._iobuf.write(message) + c.process_io_buffer() # make sure it errored correctly c.defunct.assert_called_once_with(ANY) @@ -110,9 +98,10 @@ def test_negative_body_length(self, *args): # read in a SupportedMessage response header = self.make_header_prefix(SupportedMessage) - options = self.make_options_body() - message = self.make_msg(header, options) - c.process_msg(message, -13) + message = header + int32_pack(-13) + c._iobuf = BytesIO() + c._iobuf.write(message) + c.process_io_buffer() # make sure it errored correctly c.defunct.assert_called_once_with(ANY) @@ -135,8 +124,7 @@ def test_unsupported_cql_version(self, *args): }) options = options_buf.getvalue() - message = self.make_msg(header, options) - c.process_msg(message, len(message) - 8) + c.process_msg(_Frame(version=4, flags=0, stream=0, opcode=SupportedMessage.opcode, body_offset=9, end_pos=9 + len(options)), options) # make sure it errored correctly c.defunct.assert_called_once_with(ANY) @@ -155,8 +143,6 @@ def test_prefer_lz4_compression(self, *args): locally_supported_compressions['snappy'] = ('snappycompress', 'snappydecompress') # read in a SupportedMessage response - header = self.make_header_prefix(SupportedMessage) - options_buf = BytesIO() write_stringmultimap(options_buf, { 'CQL_VERSION': ['3.0.3'], @@ -164,8 +150,7 @@ def test_prefer_lz4_compression(self, *args): }) options = options_buf.getvalue() - message = self.make_msg(header, options) - c.process_msg(message, len(message) - 8) + c.process_msg(_Frame(version=4, flags=0, stream=0, opcode=SupportedMessage.opcode, body_offset=9, end_pos=9 + len(options)), options) self.assertEqual(c.decompressor, locally_supported_compressions['lz4'][1]) @@ -192,8 +177,7 @@ def test_requested_compression_not_available(self, *args): }) options = options_buf.getvalue() - message = self.make_msg(header, options) - c.process_msg(message, len(message) - 8) + c.process_msg(_Frame(version=4, flags=0, stream=0, opcode=SupportedMessage.opcode, body_offset=9, end_pos=9 + len(options)), options) # make sure it errored correctly c.defunct.assert_called_once_with(ANY) @@ -223,8 +207,7 @@ def test_use_requested_compression(self, *args): }) options = options_buf.getvalue() - message = self.make_msg(header, options) - c.process_msg(message, len(message) - 8) + c.process_msg(_Frame(version=4, flags=0, stream=0, opcode=SupportedMessage.opcode, body_offset=9, end_pos=9 + len(options)), options) self.assertEqual(c.decompressor, locally_supported_compressions['snappy'][1]) From 7abad89169c213e1edc1e9cc03625afa3aa041d6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 9 Jun 2015 12:21:07 -0500 Subject: [PATCH 0194/2431] Change return conversion for murmur3 ext PYTHON-331 Fixes an issue where the hash value was cast to a four-byte type on some platforms. --- cassandra/murmur3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/murmur3.c b/cassandra/murmur3.c index bdcb97289f..657c01f69d 100644 --- a/cassandra/murmur3.c +++ b/cassandra/murmur3.c @@ -201,7 +201,7 @@ murmur3(PyObject *self, PyObject *args) // TODO handle x86 version? result = MurmurHash3_x64_128((void *)key, len, seed); - return (PyObject *) PyLong_FromLong((long int)result); + return (PyObject *) PyLong_FromLongLong(result); } static PyMethodDef murmur3_methods[] = { From 57f3b0f6a70d7fe7cf0c8272301e8cee9d1dc244 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 10 Jun 2015 10:14:05 -0500 Subject: [PATCH 0195/2431] Don't default to TokenAware LBP is murmur3 ext. is not present. --- cassandra/cluster.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cb2502e7d2..834872398b 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -61,7 +61,7 @@ BatchMessage, RESULT_KIND_PREPARED, RESULT_KIND_SET_KEYSPACE, RESULT_KIND_ROWS, RESULT_KIND_SCHEMA_CHANGE) -from cassandra.metadata import Metadata, protect_name +from cassandra.metadata import Metadata, protect_name, murmur3 from cassandra.policies import (TokenAwarePolicy, DCAwareRoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, RetryPolicy) @@ -173,7 +173,9 @@ def _shutdown_cluster(cluster): import platform if platform.python_implementation() == 'CPython': def default_lbp_factory(): - return TokenAwarePolicy(DCAwareRoundRobinPolicy()) + if murmur3 is not None: + return TokenAwarePolicy(DCAwareRoundRobinPolicy()) + return DCAwareRoundRobinPolicy() else: def default_lbp_factory(): return DCAwareRoundRobinPolicy() From 0ca97eaf4335f7a89b133ecdcfe9e7dab1419733 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 10 Jun 2015 10:14:36 -0500 Subject: [PATCH 0196/2431] cqle: deprecate cqle.columns.TimeUUID.from_datetime Prefer utility function in core instead. Resolves an issue with rounding in the cqle function, manifesting on Windows PYTHON-341 --- cassandra/cqlengine/columns.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index d418b11947..353064b6bc 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -558,30 +558,10 @@ def from_datetime(self, dt): :type dt: datetime :return: """ - global _last_timestamp - - epoch = datetime(1970, 1, 1, tzinfo=dt.tzinfo) - offset = epoch.tzinfo.utcoffset(epoch).total_seconds() if epoch.tzinfo else 0 - timestamp = (dt - epoch).total_seconds() - offset - - node = None - clock_seq = None - - nanoseconds = int(timestamp * 1e9) - timestamp = int(nanoseconds // 100) + 0x01b21dd213814000 - - if clock_seq is None: - import random - clock_seq = random.randrange(1 << 14) # instead of stable storage - time_low = timestamp & 0xffffffff - time_mid = (timestamp >> 32) & 0xffff - time_hi_version = (timestamp >> 48) & 0x0fff - clock_seq_low = clock_seq & 0xff - clock_seq_hi_variant = (clock_seq >> 8) & 0x3f - if node is None: - node = getnode() - return pyUUID(fields=(time_low, time_mid, time_hi_version, - clock_seq_hi_variant, clock_seq_low, node), version=1) + msg = "cqlengine.columns.TimeUUID.from_datetime is deprecated. Use cassandra.util.uuid_from_time instead." + warnings.warn(msg, DeprecationWarning) + log.warning(msg) + return util.uuid_from_time(dt) class Boolean(Column): From 04b69b96d348e9313d6c4727719ea2c5f98cdc8b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 10 Jun 2015 11:18:50 -0500 Subject: [PATCH 0197/2431] Don't wait for schema agreement during startup. Fixes an issue where connect will stall when connecting to mixed version clusters. Should not affect model consistency since we are already registered for schema_change events. --- cassandra/cluster.py | 2 +- tests/integration/standard/test_cluster.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cb2502e7d2..3f72e77455 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2118,7 +2118,7 @@ def _try_connect(self, host): peers_query, local_query, timeout=self._timeout) self._refresh_node_list_and_token_map(connection, preloaded_results=shared_results) - self._refresh_schema(connection, preloaded_results=shared_results) + self._refresh_schema(connection, preloaded_results=shared_results, schema_agreement_wait=-1) if not self._cluster.metadata.keyspaces: log.warning("[control connection] No schema built on connect; retrying without wait for schema agreement") self._refresh_schema(connection, preloaded_results=shared_results, schema_agreement_wait=0) diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index bf6bb1d476..f45a32ed43 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -336,10 +336,7 @@ def test_refresh_schema_no_wait(self): # cluster agreement wait exceeded c = Cluster(protocol_version=PROTOCOL_VERSION, max_schema_agreement_wait=agreement_timeout) - start_time = time.time() - s = c.connect() - end_time = time.time() - self.assertGreaterEqual(end_time - start_time, agreement_timeout) + c.connect() self.assertTrue(c.metadata.keyspaces) # cluster agreement wait used for refresh From 434014e355ae3a4a8ba24838c131786ae7fb4645 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 10 Jun 2015 17:14:18 -0500 Subject: [PATCH 0198/2431] Refactor socket creation to Connection base PYTHON-322 Fixes an issue where AsyncoreConnection could not be used with SSL in Python 2.6 (ssl did not implement connect_ex). Refactored socket connect to common routine, and simplified AsyncoreConnection by using that and initializing the dispatcher with the connected socket. Also adds ssl support to EventletConnection. --- cassandra/connection.py | 38 +++++++++++++++++++ cassandra/io/asyncorereactor.py | 65 +++------------------------------ cassandra/io/eventletreactor.py | 32 +++------------- cassandra/io/geventreactor.py | 27 +++----------- cassandra/io/libevreactor.py | 32 +--------------- cassandra/io/twistedreactor.py | 1 - 6 files changed, 55 insertions(+), 140 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index c239313c45..2cf1c21949 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -19,10 +19,16 @@ import io import logging import os +import socket import sys from threading import Thread, Event, RLock import time +try: + import ssl +except ImportError: + ssl = None # NOQA + if 'gevent.monkey' in sys.modules: from gevent.queue import Queue, Empty else: @@ -177,6 +183,11 @@ class Connection(object): is_control_connection = False _iobuf = None + _socket = None + + _socket_impl = socket + _ssl_impl = ssl + def __init__(self, host='127.0.0.1', port=9042, authenticator=None, ssl_options=None, sockopts=None, compression=True, cql_version=None, protocol_version=2, is_control_connection=False, @@ -256,6 +267,33 @@ def factory(cls, host, timeout, *args, **kwargs): else: return conn + def _connect_socket(self): + sockerr = None + addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) + for (af, socktype, proto, canonname, sockaddr) in addresses: + try: + self._socket = self._socket_impl.socket(af, socktype, proto) + if self.ssl_options: + if not self._ssl_impl: + raise Exception("This version of Python was not compiled with SSL support") + self._socket = self._ssl_impl.wrap_socket(self._socket, **self.ssl_options) + self._socket.settimeout(1.0) + self._socket.connect(sockaddr) + sockerr = None + break + except socket.error as err: + if self._socket: + self._socket.close() + self._socket = None + sockerr = err + + if sockerr: + raise socket.error(sockerr.errno, "Tried connecting to %s. Last error: %s" % ([a[4] for a in addresses], sockerr.strerror)) + + if self.sockopts: + for args in self.sockopts: + self._socket.setsockopt(*args) + def close(self): raise NotImplementedError() diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index ef687c388c..235c7660b5 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -23,7 +23,6 @@ from six.moves import range -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN, errorcode try: from weakref import WeakSet except ImportError: @@ -36,9 +35,7 @@ except ImportError: ssl = None # NOQA -from cassandra import OperationTimedOut -from cassandra.connection import (Connection, ConnectionShutdown, - ConnectionException, NONBLOCKING) +from cassandra.connection import (Connection, ConnectionShutdown, NONBLOCKING) from cassandra.protocol import RegisterMessage log = logging.getLogger(__name__) @@ -168,66 +165,17 @@ def __init__(self, *args, **kwargs): self._loop.connection_created(self) - sockerr = None - addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) - for (af, socktype, proto, canonname, sockaddr) in addresses: - try: - self.create_socket(af, socktype) - self.connect(sockaddr) - sockerr = None - break - except socket.error as err: - sockerr = err - if sockerr: - raise socket.error(sockerr.errno, "Tried connecting to %s. Last error: %s" % ([a[4] for a in addresses], sockerr.strerror)) - - self.add_channel() - - if self.sockopts: - for args in self.sockopts: - self.socket.setsockopt(*args) + self._connect_socket() + asyncore.dispatcher.__init__(self, self._socket) self._writable = True self._readable = True + self._send_options_message() + # start the event loop if needed self._loop.maybe_start() - def set_socket(self, sock): - # Overrides the same method in asyncore. We deliberately - # do not call add_channel() in this method so that we can call - # it later, after connect() has completed. - self.socket = sock - self._fileno = sock.fileno() - - def create_socket(self, family, type): - # copied from asyncore, but with the line to set the socket in - # non-blocking mode removed (we will do that after connecting) - self.family_and_type = family, type - sock = socket.socket(family, type) - if self.ssl_options: - if not ssl: - raise Exception("This version of Python was not compiled with SSL support") - sock = ssl.wrap_socket(sock, **self.ssl_options) - self.set_socket(sock) - - def connect(self, address): - # this is copied directly from asyncore.py, except that - # a timeout is set before connecting - self.connected = False - self.connecting = True - self.socket.settimeout(1.0) - err = self.socket.connect_ex(address) - if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ - or err == EINVAL and os.name in ('nt', 'ce'): - raise ConnectionException("Timed out connecting to %s" % (address[0])) - if err in (0, EISCONN): - self.addr = address - self.socket.setblocking(0) - self.handle_connect_event() - else: - raise socket.error(err, os.strerror(err)) - def close(self): with self.lock: if self.is_closed: @@ -248,9 +196,6 @@ def close(self): # don't leave in-progress operations hanging self.connected_event.set() - def handle_connect(self): - self._send_options_message() - def handle_error(self): self.defunct(sys.exc_info()[1]) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index 670d0f1865..484ebbe3fb 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -24,11 +24,9 @@ from functools import partial import logging import os -from threading import Event - from six.moves import xrange +from threading import Event -from cassandra import OperationTimedOut from cassandra.connection import Connection, ConnectionShutdown from cassandra.protocol import RegisterMessage @@ -51,7 +49,9 @@ class EventletConnection(Connection): _total_reqd_bytes = 0 _read_watcher = None _write_watcher = None - _socket = None + + _socket_impl = eventlet.green.socket + _ssl_impl = eventlet.green.ssl @classmethod def initialize_reactor(cls): @@ -66,29 +66,7 @@ def __init__(self, *args, **kwargs): self._callbacks = {} self._push_watchers = defaultdict(set) - sockerr = None - addresses = socket.getaddrinfo( - self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM - ) - for (af, socktype, proto, canonname, sockaddr) in addresses: - try: - self._socket = socket.socket(af, socktype, proto) - self._socket.settimeout(1.0) - self._socket.connect(sockaddr) - sockerr = None - break - except socket.error as err: - sockerr = err - if sockerr: - raise socket.error( - sockerr.errno, - "Tried connecting to %s. Last error: %s" % ( - [a[4] for a in addresses], sockerr.strerror) - ) - - if self.sockopts: - for args in self.sockopts: - self._socket.setsockopt(*args) + self._connect_socket() self._read_watcher = eventlet.spawn(lambda: self.handle_read()) self._write_watcher = eventlet.spawn(lambda: self.handle_write()) diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 6e9af0da4d..e9f1dd7ceb 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import gevent -from gevent import select, socket, ssl +from gevent import select, socket from gevent.event import Event from gevent.queue import Queue @@ -25,7 +25,6 @@ from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -from cassandra import OperationTimedOut from cassandra.connection import Connection, ConnectionShutdown from cassandra.protocol import RegisterMessage @@ -48,7 +47,9 @@ class GeventConnection(Connection): _total_reqd_bytes = 0 _read_watcher = None _write_watcher = None - _socket = None + + _socket_impl = gevent.socket + _ssl_impl = gevent.ssl def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) @@ -59,25 +60,7 @@ def __init__(self, *args, **kwargs): self._callbacks = {} self._push_watchers = defaultdict(set) - sockerr = None - addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) - for (af, socktype, proto, canonname, sockaddr) in addresses: - try: - self._socket = socket.socket(af, socktype, proto) - if self.ssl_options: - self._socket = ssl.wrap_socket(self._socket, **self.ssl_options) - self._socket.settimeout(1.0) - self._socket.connect(sockaddr) - sockerr = None - break - except socket.error as err: - sockerr = err - if sockerr: - raise socket.error(sockerr.errno, "Tried connecting to %s. Last error: %s" % ([a[4] for a in addresses], sockerr.strerror)) - - if self.sockopts: - for args in self.sockopts: - self._socket.setsockopt(*args) + self._connect_socket() self._read_watcher = gevent.spawn(self.handle_read) self._write_watcher = gevent.spawn(self.handle_write) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 93b4c97854..eaa6f4ae38 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -22,8 +22,7 @@ from six.moves import xrange -from cassandra import OperationTimedOut -from cassandra.connection import Connection, ConnectionShutdown, NONBLOCKING +from cassandra.connection import Connection, ConnectionShutdown, NONBLOCKING, ssl from cassandra.protocol import RegisterMessage try: import cassandra.io.libevwrapper as libev @@ -37,11 +36,6 @@ "the C extension.") -try: - import ssl -except ImportError: - ssl = None # NOQA - log = logging.getLogger(__name__) @@ -244,31 +238,9 @@ def __init__(self, *args, **kwargs): self._callbacks = {} self.deque = deque() self._deque_lock = Lock() - - sockerr = None - addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) - for (af, socktype, proto, canonname, sockaddr) in addresses: - try: - self._socket = socket.socket(af, socktype, proto) - if self.ssl_options: - if not ssl: - raise Exception("This version of Python was not compiled with SSL support") - self._socket = ssl.wrap_socket(self._socket, **self.ssl_options) - self._socket.settimeout(1.0) # TODO potentially make this value configurable - self._socket.connect(sockaddr) - sockerr = None - break - except socket.error as err: - sockerr = err - if sockerr: - raise socket.error(sockerr.errno, "Tried connecting to %s. Last error: %s" % ([a[4] for a in addresses], sockerr.strerror)) - + self._connect_socket() self._socket.setblocking(0) - if self.sockopts: - for args in self.sockopts: - self._socket.setsockopt(*args) - with self._libevloop._lock: self._read_watcher = libev.IO(self._socket.fileno(), libev.EV_READ, self._libevloop._loop, self.handle_read) self._write_watcher = libev.IO(self._socket.fileno(), libev.EV_WRITE, self._libevloop._loop, self.handle_write) diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index ff81e5613f..27d4f7a471 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -22,7 +22,6 @@ import weakref import atexit -from cassandra import OperationTimedOut from cassandra.connection import Connection, ConnectionShutdown from cassandra.protocol import RegisterMessage From 2b3a72efcc47479a7bd3adef3a522d6550047ae1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 11 Jun 2015 11:47:08 -0500 Subject: [PATCH 0199/2431] cqle: use integer division for converting timedelta to seconds Emulates the timedelta.total_seconds() function in python 2.7+ --- cassandra/cqlengine/functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cassandra/cqlengine/functions.py b/cassandra/cqlengine/functions.py index 43c98afea5..d69e16f1fe 100644 --- a/cassandra/cqlengine/functions.py +++ b/cassandra/cqlengine/functions.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import division from datetime import datetime from cassandra.cqlengine import UnicodeMixin, ValidationError @@ -23,7 +24,9 @@ def get_total_seconds(td): return td.total_seconds() else: def get_total_seconds(td): - return 86400*td.days + td.seconds + td.microseconds/1e6 + # integer division used here to emulate built-in total_seconds + return ((86400 * td.days + td.seconds) * 10 ** 6 + td.microseconds) / 10 ** 6 + class QueryValue(UnicodeMixin): """ From b47cbd627924fc574660fa81c67b175f733b98d1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 11 Jun 2015 11:49:34 -0500 Subject: [PATCH 0200/2431] cqle: Update docs reflecting new 2.6 compatibility --- README.rst | 2 -- docs/index.rst | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 110505da97..bbb63a323b 100644 --- a/README.rst +++ b/README.rst @@ -8,8 +8,6 @@ A modern, `feature-rich `_ a The driver supports Python 2.6, 2.7, 3.3, and 3.4*. -\* cqlengine component presently supports Python 2.6+ - Feedback Requested ------------------ **Help us focus our efforts!** Provide your input on the `Platform and Runtime Survey `_ (we kept it short). diff --git a/docs/index.rst b/docs/index.rst index 0b89c369da..7775dc62b4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ A Python client driver for `Apache Cassandra `_. This driver works exclusively with the Cassandra Query Language v3 (CQL3) and Cassandra's native protocol. Cassandra 1.2+ is supported. -The core driver supports Python 2.6, 2.7, 3.3, and 3.4. The object mapper is presently supported in 2.7+. +The driver supports Python 2.6, 2.7, 3.3, and 3.4. This driver is open source under the `Apache v2 License `_. From 2f6d11208e76bcf8975f4b08129254b2bc165e24 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 11 Jun 2015 16:33:20 -0500 Subject: [PATCH 0201/2431] cqle: Make integration tests run in Python 2.6 PYTHON-288 --- cassandra/cqlengine/columns.py | 4 +- cassandra/cqlengine/functions.py | 39 ++---- tests/integration/cqlengine/base.py | 12 +- .../columns/test_container_columns.py | 113 ++++++++---------- .../cqlengine/columns/test_counter_column.py | 17 ++- .../cqlengine/columns/test_static_column.py | 14 ++- .../cqlengine/columns/test_validation.py | 26 ++-- .../cqlengine/columns/test_value_io.py | 70 ++++++----- .../cqlengine/connections/__init__.py | 14 --- .../management/test_compaction_settings.py | 7 +- .../cqlengine/management/test_management.py | 8 +- .../model/test_equality_operations.py | 1 - .../integration/cqlengine/model/test_model.py | 10 +- .../cqlengine/model/test_model_io.py | 5 +- .../integration/cqlengine/model/test_udts.py | 5 +- .../cqlengine/operators/test_base_operator.py | 8 +- .../cqlengine/query/test_batch_query.py | 2 +- .../cqlengine/query/test_datetime_queries.py | 3 +- .../cqlengine/query/test_queryoperators.py | 4 +- .../cqlengine/query/test_queryset.py | 10 +- .../cqlengine/query/test_updates.py | 22 ++-- .../statements/test_assignment_clauses.py | 51 ++++---- .../statements/test_assignment_statement.py | 7 +- .../statements/test_base_statement.py | 7 +- .../statements/test_insert_statement.py | 7 +- .../statements/test_select_statement.py | 7 +- .../statements/test_update_statement.py | 10 +- .../cqlengine/statements/test_where_clause.py | 7 +- .../integration/cqlengine/test_batch_query.py | 2 +- .../integration/cqlengine/test_ifnotexists.py | 9 +- tests/integration/cqlengine/test_load.py | 8 +- tests/integration/cqlengine/test_timestamp.py | 15 ++- .../integration/cqlengine/test_transaction.py | 7 +- tox.ini | 5 - 34 files changed, 280 insertions(+), 256 deletions(-) delete mode 100644 tests/integration/cqlengine/connections/__init__.py diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index 13dea5b430..8f8a6e2546 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -639,7 +639,7 @@ def validate(self, value): if val is None: return try: - return _Decimal(val) + return _Decimal(repr(val)) if isinstance(val, float) else _Decimal(val) except InvalidOperation: raise ValidationError("{0} '{1}' can't be coerced to decimal".format(self.column_name, val)) @@ -808,7 +808,7 @@ def validate(self, value): if not isinstance(val, dict): raise ValidationError('{0} {1} is not a dict object'.format(self.column_name, val)) if None in val: - raise ValidationError("{} None is not allowed in a map".format(self.column_name)) + raise ValidationError("{0} None is not allowed in a map".format(self.column_name)) return dict((self.key_col.validate(k), self.value_col.validate(v)) for k, v in val.items()) def to_python(self, value): diff --git a/cassandra/cqlengine/functions.py b/cassandra/cqlengine/functions.py index d69e16f1fe..bc210d69ce 100644 --- a/cassandra/cqlengine/functions.py +++ b/cassandra/cqlengine/functions.py @@ -62,23 +62,16 @@ class BaseQueryFunction(QueryValue): pass -class MinTimeUUID(BaseQueryFunction): - """ - return a fake timeuuid corresponding to the smallest possible timeuuid for the given timestamp - - http://cassandra.apache.org/doc/cql3/CQL.html#timeuuidFun - """ - - format_string = 'MinTimeUUID(%({0})s)' +class TimeUUIDQueryFunction(BaseQueryFunction): def __init__(self, value): """ - :param value: the time to create a maximum time uuid from + :param value: the time to create bounding time uuid from :type value: datetime """ if not isinstance(value, datetime): raise ValidationError('datetime instance is required') - super(MinTimeUUID, self).__init__(value) + super(TimeUUIDQueryFunction, self).__init__(value) def to_database(self, val): epoch = datetime(1970, 1, 1, tzinfo=val.tzinfo) @@ -89,31 +82,23 @@ def update_context(self, ctx): ctx[str(self.context_id)] = self.to_database(self.value) -class MaxTimeUUID(BaseQueryFunction): +class MinTimeUUID(TimeUUIDQueryFunction): """ - return a fake timeuuid corresponding to the largest possible timeuuid for the given timestamp + return a fake timeuuid corresponding to the smallest possible timeuuid for the given timestamp http://cassandra.apache.org/doc/cql3/CQL.html#timeuuidFun """ + format_string = 'MinTimeUUID(%({0})s)' - format_string = 'MaxTimeUUID(%({0})s)' - def __init__(self, value): - """ - :param value: the time to create a minimum time uuid from - :type value: datetime - """ - if not isinstance(value, datetime): - raise ValidationError('datetime instance is required') - super(MaxTimeUUID, self).__init__(value) +class MaxTimeUUID(TimeUUIDQueryFunction): + """ + return a fake timeuuid corresponding to the largest possible timeuuid for the given timestamp - def to_database(self, val): - epoch = datetime(1970, 1, 1, tzinfo=val.tzinfo) - offset = epoch.tzinfo.utcoffset(epoch).total_seconds() if epoch.tzinfo else 0 - return int(((val - epoch).total_seconds() - offset) * 1000) + http://cassandra.apache.org/doc/cql3/CQL.html#timeuuidFun + """ + format_string = 'MaxTimeUUID(%({0})s)' - def update_context(self, ctx): - ctx[str(self.context_id)] = self.to_database(self.value) class Token(BaseQueryFunction): diff --git a/tests/integration/cqlengine/base.py b/tests/integration/cqlengine/base.py index f0b75f5bd1..856f2bac4f 100644 --- a/tests/integration/cqlengine/base.py +++ b/tests/integration/cqlengine/base.py @@ -11,28 +11,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa import sys -from unittest import TestCase from cassandra.cqlengine.connection import get_session -class BaseCassEngTestCase(TestCase): +class BaseCassEngTestCase(unittest.TestCase): session = None def setUp(self): self.session = get_session() - super(BaseCassEngTestCase, self).setUp() def assertHasAttr(self, obj, attr): self.assertTrue(hasattr(obj, attr), - "{} doesn't have attribute: {}".format(obj, attr)) + "{0} doesn't have attribute: {1}".format(obj, attr)) def assertNotHasAttr(self, obj, attr): self.assertFalse(hasattr(obj, attr), - "{} shouldn't have the attribute: {}".format(obj, attr)) + "{0} shouldn't have the attribute: {1}".format(obj, attr)) if sys.version_info > (3, 0): def assertItemsEqual(self, first, second, msg=None): diff --git a/tests/integration/cqlengine/columns/test_container_columns.py b/tests/integration/cqlengine/columns/test_container_columns.py index 6ec301eb67..213c625c66 100644 --- a/tests/integration/cqlengine/columns/test_container_columns.py +++ b/tests/integration/cqlengine/columns/test_container_columns.py @@ -13,13 +13,18 @@ # limitations under the License. from datetime import datetime, timedelta -import json, six, sys, traceback, logging +import json +import logging +import six +import sys +import traceback from uuid import uuid4 from cassandra import WriteTimeout -from cassandra.cqlengine.models import Model, ValidationError import cassandra.cqlengine.columns as columns +from cassandra.cqlengine.functions import get_total_seconds +from cassandra.cqlengine.models import Model, ValidationError from cassandra.cqlengine.management import sync_table, drop_table from tests.integration.cqlengine import is_prepend_reversed from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -38,14 +43,16 @@ class JsonTestColumn(columns.Column): db_type = 'text' def to_python(self, value): - if value is None: return + if value is None: + return if isinstance(value, six.string_types): return json.loads(value) else: return value def to_database(self, value): - if value is None: return + if value is None: + return return json.dumps(value) @@ -53,18 +60,15 @@ class TestSetColumn(BaseCassEngTestCase): @classmethod def setUpClass(cls): - super(TestSetColumn, cls).setUpClass() drop_table(TestSetModel) sync_table(TestSetModel) @classmethod def tearDownClass(cls): - super(TestSetColumn, cls).tearDownClass() drop_table(TestSetModel) def test_add_none_fails(self): - with self.assertRaises(ValidationError): - m = TestSetModel.create(int_set=set([None])) + self.assertRaises(ValidationError, TestSetModel.create, **{'int_set': set([None])}) def test_empty_set_initial(self): """ @@ -83,7 +87,7 @@ def test_deleting_last_item_should_succeed(self): m.save() m = TestSetModel.get(partition=m.partition) - self.assertNotIn(5, m.int_set) + self.assertTrue(5 not in m.int_set) def test_blind_deleting_last_item_should_succeed(self): m = TestSetModel.create() @@ -93,7 +97,7 @@ def test_blind_deleting_last_item_should_succeed(self): TestSetModel.objects(partition=m.partition).update(int_set=set()) m = TestSetModel.get(partition=m.partition) - self.assertNotIn(5, m.int_set) + self.assertTrue(5 not in m.int_set) def test_empty_set_retrieval(self): m = TestSetModel.create() @@ -102,7 +106,7 @@ def test_empty_set_retrieval(self): def test_io_success(self): """ Tests that a basic usage works as expected """ - m1 = TestSetModel.create(int_set={1, 2}, text_set={'kai', 'andreas'}) + m1 = TestSetModel.create(int_set=set((1, 2)), text_set=set(('kai', 'andreas'))) m2 = TestSetModel.get(partition=m1.partition) assert isinstance(m2.int_set, set) @@ -118,8 +122,7 @@ def test_type_validation(self): """ Tests that attempting to use the wrong types will raise an exception """ - with self.assertRaises(ValidationError): - TestSetModel.create(int_set={'string', True}, text_set={1, 3.0}) + self.assertRaises(ValidationError, TestSetModel.create, **{'int_set': set(('string', True)), 'text_set': set((1, 3.0))}) def test_element_count_validation(self): """ @@ -127,27 +130,26 @@ def test_element_count_validation(self): """ while True: try: - TestSetModel.create(text_set={str(uuid4()) for i in range(65535)}) + TestSetModel.create(text_set=set(str(uuid4()) for i in range(65535))) break except WriteTimeout: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb - with self.assertRaises(ValidationError): - TestSetModel.create(text_set={str(uuid4()) for i in range(65536)}) + self.assertRaises(ValidationError, TestSetModel.create, **{'text_set': set(str(uuid4()) for i in range(65536))}) def test_partial_updates(self): """ Tests that partial udpates work as expected """ - m1 = TestSetModel.create(int_set={1, 2, 3, 4}) + m1 = TestSetModel.create(int_set=set((1, 2, 3, 4))) m1.int_set.add(5) m1.int_set.remove(1) - assert m1.int_set == {2, 3, 4, 5} + assert m1.int_set == set((2, 3, 4, 5)) m1.save() m2 = TestSetModel.get(partition=m1.partition) - assert m2.int_set == {2, 3, 4, 5} + assert m2.int_set == set((2, 3, 4, 5)) def test_instantiation_with_column_class(self): """ @@ -167,9 +169,9 @@ def test_instantiation_with_column_instance(self): def test_to_python(self): """ Tests that to_python of value column is called """ column = columns.Set(JsonTestColumn) - val = {1, 2, 3} + val = set((1, 2, 3)) db_val = column.to_database(val) - assert db_val == {json.dumps(v) for v in val} + assert db_val == set(json.dumps(v) for v in val) py_val = column.to_python(db_val) assert py_val == val @@ -177,17 +179,16 @@ def test_default_empty_container_saving(self): """ tests that the default empty container is not saved if it hasn't been updated """ pkey = uuid4() # create a row with set data - TestSetModel.create(partition=pkey, int_set={3, 4}) + TestSetModel.create(partition=pkey, int_set=set((3, 4))) # create another with no set data TestSetModel.create(partition=pkey) m = TestSetModel.get(partition=pkey) - self.assertEqual(m.int_set, {3, 4}) + self.assertEqual(m.int_set, set((3, 4))) class TestListModel(Model): - partition = columns.UUID(primary_key=True, default=uuid4) int_list = columns.List(columns.Integer, required=False) text_list = columns.List(columns.Text, required=False) @@ -197,20 +198,18 @@ class TestListColumn(BaseCassEngTestCase): @classmethod def setUpClass(cls): - super(TestListColumn, cls).setUpClass() drop_table(TestListModel) sync_table(TestListModel) @classmethod def tearDownClass(cls): - super(TestListColumn, cls).tearDownClass() drop_table(TestListModel) def test_initial(self): tmp = TestListModel.create() tmp.int_list.append(1) - def test_initial(self): + def test_initial_retrieve(self): tmp = TestListModel.create() tmp2 = TestListModel.get(partition=tmp.partition) tmp2.int_list.append(1) @@ -236,8 +235,7 @@ def test_type_validation(self): """ Tests that attempting to use the wrong types will raise an exception """ - with self.assertRaises(ValidationError): - TestListModel.create(int_list=['string', True], text_list=[1, 3.0]) + self.assertRaises(ValidationError, TestListModel.create, **{'int_list': ['string', True], 'text_list': [1, 3.0]}) def test_element_count_validation(self): """ @@ -251,8 +249,7 @@ def test_element_count_validation(self): ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb - with self.assertRaises(ValidationError): - TestListModel.create(text_list=[str(uuid4()) for i in range(65536)]) + self.assertRaises(ValidationError, TestListModel.create, **{'text_list': [str(uuid4()) for _ in range(65536)]}) def test_partial_updates(self): """ Tests that partial udpates work as expected """ @@ -300,16 +297,16 @@ def test_default_empty_container_saving(self): """ tests that the default empty container is not saved if it hasn't been updated """ pkey = uuid4() # create a row with list data - TestListModel.create(partition=pkey, int_list=[1,2,3,4]) + TestListModel.create(partition=pkey, int_list=[1, 2, 3, 4]) # create another with no list data TestListModel.create(partition=pkey) m = TestListModel.get(partition=pkey) - self.assertEqual(m.int_list, [1,2,3,4]) + self.assertEqual(m.int_list, [1, 2, 3, 4]) def test_remove_entry_works(self): pkey = uuid4() - tmp = TestListModel.create(partition=pkey, int_list=[1,2]) + tmp = TestListModel.create(partition=pkey, int_list=[1, 2]) tmp.int_list.pop() tmp.update() tmp = TestListModel.get(partition=pkey) @@ -317,7 +314,7 @@ def test_remove_entry_works(self): def test_update_from_non_empty_to_empty(self): pkey = uuid4() - tmp = TestListModel.create(partition=pkey, int_list=[1,2]) + tmp = TestListModel.create(partition=pkey, int_list=[1, 2]) tmp.int_list = [] tmp.update() @@ -326,8 +323,7 @@ def test_update_from_non_empty_to_empty(self): def test_insert_none(self): pkey = uuid4() - with self.assertRaises(ValidationError): - TestListModel.create(partition=pkey, int_list=[None]) + self.assertRaises(ValidationError, TestListModel.create, **{'partition': pkey, 'int_list': [None]}) def test_blind_list_updates_from_none(self): """ Tests that updates from None work as expected """ @@ -344,6 +340,7 @@ def test_blind_list_updates_from_none(self): m3 = TestListModel.get(partition=m.partition) assert m3.int_list == [] + class TestMapModel(Model): partition = columns.UUID(primary_key=True, default=uuid4) int_map = columns.Map(columns.Integer, columns.UUID, required=False) @@ -354,13 +351,11 @@ class TestMapColumn(BaseCassEngTestCase): @classmethod def setUpClass(cls): - super(TestMapColumn, cls).setUpClass() drop_table(TestMapModel) sync_table(TestMapModel) @classmethod def tearDownClass(cls): - super(TestMapColumn, cls).tearDownClass() drop_table(TestMapModel) def test_empty_default(self): @@ -368,8 +363,7 @@ def test_empty_default(self): tmp.int_map['blah'] = 1 def test_add_none_as_map_key(self): - with self.assertRaises(ValidationError): - TestMapModel.create(int_map={None: uuid4()}) + self.assertRaises(ValidationError, TestMapModel.create, **{'int_map': {None: uuid4()}}) def test_empty_retrieve(self): tmp = TestMapModel.create() @@ -384,7 +378,7 @@ def test_remove_last_entry_works(self): tmp.save() tmp = TestMapModel.get(partition=tmp.partition) - self.assertNotIn("blah", tmp.int_map) + self.assertTrue("blah" not in tmp.int_map) def test_io_success(self): """ Tests that a basic usage works as expected """ @@ -395,25 +389,24 @@ def test_io_success(self): m1 = TestMapModel.create(int_map={1: k1, 2: k2}, text_map={'now': now, 'then': then}) m2 = TestMapModel.get(partition=m1.partition) - assert isinstance(m2.int_map, dict) - assert isinstance(m2.text_map, dict) + self.assertTrue(isinstance(m2.int_map, dict)) + self.assertTrue(isinstance(m2.text_map, dict)) - assert 1 in m2.int_map - assert 2 in m2.int_map - assert m2.int_map[1] == k1 - assert m2.int_map[2] == k2 + self.assertTrue(1 in m2.int_map) + self.assertTrue(2 in m2.int_map) + self.assertEqual(m2.int_map[1], k1) + self.assertEqual(m2.int_map[2], k2) - assert 'now' in m2.text_map - assert 'then' in m2.text_map - assert (now - m2.text_map['now']).total_seconds() < 0.001 - assert (then - m2.text_map['then']).total_seconds() < 0.001 + self.assertTrue('now' in m2.text_map) + self.assertTrue('then' in m2.text_map) + self.assertAlmostEqual(get_total_seconds(now - m2.text_map['now']), 0, 2) + self.assertAlmostEqual(get_total_seconds(then - m2.text_map['then']), 0, 2) def test_type_validation(self): """ Tests that attempting to use the wrong types will raise an exception """ - with self.assertRaises(ValidationError): - TestMapModel.create(int_map={'key': 2, uuid4(): 'val'}, text_map={2: 5}) + self.assertRaises(ValidationError, TestMapModel.create, **{'int_map': {'key': 2, uuid4(): 'val'}, 'text_map': {2: 5}}) def test_element_count_validation(self): """ @@ -421,19 +414,18 @@ def test_element_count_validation(self): """ while True: try: - TestMapModel.create(text_map={str(uuid4()): i for i in range(65535)}) + TestMapModel.create(text_map=dict((str(uuid4()), i) for i in range(65535))) break except WriteTimeout: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb - with self.assertRaises(ValidationError): - TestMapModel.create(text_map={str(uuid4()): i for i in range(65536)}) + self.assertRaises(ValidationError, TestMapModel.create, **{'text_map': dict((str(uuid4()), i) for i in range(65536))}) def test_partial_updates(self): """ Tests that partial udpates work as expected """ now = datetime.now() - #derez it a bit + # derez it a bit now = datetime(*now.timetuple()[:-3]) early = now - timedelta(minutes=30) earlier = early - timedelta(minutes=30) @@ -511,7 +503,7 @@ def test_to_python(self): column = columns.Map(JsonTestColumn, JsonTestColumn) val = {1: 2, 3: 4, 5: 6} db_val = column.to_database(val) - assert db_val == {json.dumps(k):json.dumps(v) for k,v in val.items()} + assert db_val == dict((json.dumps(k), json.dumps(v)) for k, v in val.items()) py_val = column.to_python(db_val) assert py_val == val @@ -530,7 +522,6 @@ def test_default_empty_container_saving(self): class TestCamelMapModel(Model): - partition = columns.UUID(primary_key=True, default=uuid4) camelMap = columns.Map(columns.Text, columns.Integer, required=False) @@ -539,13 +530,11 @@ class TestCamelMapColumn(BaseCassEngTestCase): @classmethod def setUpClass(cls): - super(TestCamelMapColumn, cls).setUpClass() drop_table(TestCamelMapModel) sync_table(TestCamelMapModel) @classmethod def tearDownClass(cls): - super(TestCamelMapColumn, cls).tearDownClass() drop_table(TestCamelMapModel) def test_camelcase_column(self): diff --git a/tests/integration/cqlengine/columns/test_counter_column.py b/tests/integration/cqlengine/columns/test_counter_column.py index caded408cb..c9579ba7b9 100644 --- a/tests/integration/cqlengine/columns/test_counter_column.py +++ b/tests/integration/cqlengine/columns/test_counter_column.py @@ -31,41 +31,48 @@ class TestClassConstruction(BaseCassEngTestCase): def test_defining_a_non_counter_column_fails(self): """ Tests that defining a non counter column field in a model with a counter column fails """ - with self.assertRaises(ModelDefinitionException): + try: class model(Model): partition = columns.UUID(primary_key=True, default=uuid4) counter = columns.Counter() text = columns.Text() + self.fail("did not raise expected ModelDefinitionException") + except ModelDefinitionException: + pass def test_defining_a_primary_key_counter_column_fails(self): """ Tests that defining primary keys on counter columns fails """ - with self.assertRaises(TypeError): + try: class model(Model): partition = columns.UUID(primary_key=True, default=uuid4) cluster = columns.Counter(primary_ley=True) counter = columns.Counter() + self.fail("did not raise expected TypeError") + except TypeError: + pass # force it - with self.assertRaises(ModelDefinitionException): + try: class model(Model): partition = columns.UUID(primary_key=True, default=uuid4) cluster = columns.Counter() cluster.primary_key = True counter = columns.Counter() + self.fail("did not raise expected ModelDefinitionException") + except ModelDefinitionException: + pass class TestCounterColumn(BaseCassEngTestCase): @classmethod def setUpClass(cls): - super(TestCounterColumn, cls).setUpClass() drop_table(TestCounterModel) sync_table(TestCounterModel) @classmethod def tearDownClass(cls): - super(TestCounterColumn, cls).tearDownClass() drop_table(TestCounterModel) def test_updates(self): diff --git a/tests/integration/cqlengine/columns/test_static_column.py b/tests/integration/cqlengine/columns/test_static_column.py index 7037e92a50..a4864050d9 100644 --- a/tests/integration/cqlengine/columns/test_static_column.py +++ b/tests/integration/cqlengine/columns/test_static_column.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest import skipUnless +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + from uuid import uuid4 from cassandra.cqlengine import columns @@ -33,19 +37,21 @@ class TestStaticModel(Model): text = columns.Text() -@skipUnless(STATIC_SUPPORTED, "only runs against the cql3 protocol v2.0") class TestStaticColumn(BaseCassEngTestCase): + def setUp(cls): + if not STATIC_SUPPORTED: + raise unittest.SkipTest("only runs against the cql3 protocol v2.0") + super(TestStaticColumn, cls).setUp() + @classmethod def setUpClass(cls): - super(TestStaticColumn, cls).setUpClass() drop_table(TestStaticModel) if STATIC_SUPPORTED: # setup and teardown run regardless of skip sync_table(TestStaticModel) @classmethod def tearDownClass(cls): - super(TestStaticColumn, cls).tearDownClass() drop_table(TestStaticModel) def test_mixed_updates(self): diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index a74f83a0de..5426548e5e 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -12,9 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + from datetime import datetime, timedelta, date, tzinfo from decimal import Decimal as D -from unittest import TestCase, SkipTest from uuid import uuid4, uuid1 from cassandra import InvalidRequest @@ -46,12 +50,10 @@ class DatetimeTest(Model): @classmethod def setUpClass(cls): - super(TestDatetime, cls).setUpClass() sync_table(cls.DatetimeTest) @classmethod def tearDownClass(cls): - super(TestDatetime, cls).tearDownClass() drop_table(cls.DatetimeTest) def test_datetime_io(self): @@ -95,7 +97,6 @@ class BoolDefaultValueTest(Model): @classmethod def setUpClass(cls): - super(TestBoolDefault, cls).setUpClass() sync_table(cls.BoolDefaultValueTest) def test_default_is_set(self): @@ -112,7 +113,6 @@ class BoolValidationTest(Model): @classmethod def setUpClass(cls): - super(TestBoolValidation, cls).setUpClass() sync_table(cls.BoolValidationTest) def test_validation_preserves_none(self): @@ -129,12 +129,10 @@ class VarIntTest(Model): @classmethod def setUpClass(cls): - super(TestVarInt, cls).setUpClass() sync_table(cls.VarIntTest) @classmethod def tearDownClass(cls): - super(TestVarInt, cls).tearDownClass() sync_table(cls.VarIntTest) def test_varint_io(self): @@ -155,14 +153,12 @@ class DateTest(Model): @classmethod def setUpClass(cls): if PROTOCOL_VERSION < 4: - raise SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - super(TestDate, cls).setUpClass() sync_table(cls.DateTest) @classmethod def tearDownClass(cls): - super(TestDate, cls).tearDownClass() drop_table(cls.DateTest) def test_date_io(self): @@ -195,12 +191,10 @@ class DecimalTest(Model): @classmethod def setUpClass(cls): - super(TestDecimal, cls).setUpClass() sync_table(cls.DecimalTest) @classmethod def tearDownClass(cls): - super(TestDecimal, cls).tearDownClass() drop_table(cls.DecimalTest) def test_decimal_io(self): @@ -220,12 +214,10 @@ class UUIDTest(Model): @classmethod def setUpClass(cls): - super(TestUUID, cls).setUpClass() sync_table(cls.UUIDTest) @classmethod def tearDownClass(cls): - super(TestUUID, cls).tearDownClass() drop_table(cls.UUIDTest) def test_uuid_str_with_dashes(self): @@ -255,12 +247,10 @@ class TimeUUIDTest(Model): @classmethod def setUpClass(cls): - super(TestTimeUUID, cls).setUpClass() sync_table(cls.TimeUUIDTest) @classmethod def tearDownClass(cls): - super(TestTimeUUID, cls).tearDownClass() drop_table(cls.TimeUUIDTest) def test_timeuuid_io(self): @@ -361,10 +351,10 @@ def test_extra_field(self): drop_table(self.TestModel) sync_table(self.TestModel) self.TestModel.create() - execute("ALTER TABLE {} add blah int".format(self.TestModel.column_family_name(include_keyspace=True))) + execute("ALTER TABLE {0} add blah int".format(self.TestModel.column_family_name(include_keyspace=True))) self.TestModel.objects().all() -class TestTimeUUIDFromDatetime(TestCase): +class TestTimeUUIDFromDatetime(BaseCassEngTestCase): def test_conversion_specific_date(self): dt = datetime(1981, 7, 11, microsecond=555000) diff --git a/tests/integration/cqlengine/columns/test_value_io.py b/tests/integration/cqlengine/columns/test_value_io.py index 243f7096ad..3a13b5a4e0 100644 --- a/tests/integration/cqlengine/columns/test_value_io.py +++ b/tests/integration/cqlengine/columns/test_value_io.py @@ -11,12 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa from datetime import datetime, timedelta, time from decimal import Decimal from uuid import uuid1, uuid4, UUID import six -import unittest from cassandra.cqlengine import columns from cassandra.cqlengine.management import sync_table @@ -58,7 +61,7 @@ def setUpClass(cls): # create a table with the given column class IOTestModel(Model): - table_name = cls.column.db_type + "_io_test_model_{}".format(uuid4().hex[:8]) + table_name = cls.column.db_type + "_io_test_model_{0}".format(uuid4().hex[:8]) pkey = cls.column(primary_key=True) data = cls.column() @@ -150,22 +153,6 @@ class TestDateTime(BaseColumnIOTest): data_val = now + timedelta(days=1) -class TestDate(BaseColumnIOTest): - - @classmethod - def setUpClass(cls): - if PROTOCOL_VERSION < 4: - raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - - super(TestDate, cls).setUpClass() - - column = columns.Date - - now = Date(datetime.now().date()) - pkey_val = now - data_val = Date(now.days_from_epoch + 1) - - class TestUUID(BaseColumnIOTest): column = columns.UUID @@ -218,16 +205,43 @@ class TestDecimalIO(BaseColumnIOTest): data_val = Decimal('0.005'), 3.5, '8' def comparator_converter(self, val): - return Decimal(val) + return Decimal(repr(val) if isinstance(val, float) else val) -class TestTime(BaseColumnIOTest): + +class ProtocolV4Test(BaseColumnIOTest): @classmethod def setUpClass(cls): + if PROTOCOL_VERSION >= 4: + super(ProtocolV4Test, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + if PROTOCOL_VERSION >= 4: + super(ProtocolV4Test, cls).tearDownClass() + +class TestDate(ProtocolV4Test): + + def setUp(self): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + + super(TestDate, self).setUp() + + column = columns.Date + + now = Date(datetime.now().date()) + pkey_val = now + data_val = Date(now.days_from_epoch + 1) + + +class TestTime(ProtocolV4Test): + + def setUp(self): if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - super(TestTime, cls).setUpClass() + super(TestTime, self).setUp() column = columns.Time @@ -235,14 +249,13 @@ def setUpClass(cls): data_val = Time(time(16, 47, 25, 7)) -class TestSmallInt(BaseColumnIOTest): +class TestSmallInt(ProtocolV4Test): - @classmethod - def setUpClass(cls): + def setUp(self): if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - super(TestSmallInt, cls).setUpClass() + super(TestSmallInt, self).setUp() column = columns.SmallInt @@ -250,14 +263,13 @@ def setUpClass(cls): data_val = 32523 -class TestTinyInt(BaseColumnIOTest): +class TestTinyInt(ProtocolV4Test): - @classmethod - def setUpClass(cls): + def setUp(self): if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - super(TestTinyInt, cls).setUpClass() + super(TestTinyInt, self).setUp() column = columns.TinyInt diff --git a/tests/integration/cqlengine/connections/__init__.py b/tests/integration/cqlengine/connections/__init__.py deleted file mode 100644 index 55181d93fe..0000000000 --- a/tests/integration/cqlengine/connections/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2015 DataStax, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/tests/integration/cqlengine/management/test_compaction_settings.py b/tests/integration/cqlengine/management/test_compaction_settings.py index 0efd91570d..df443bf86f 100644 --- a/tests/integration/cqlengine/management/test_compaction_settings.py +++ b/tests/integration/cqlengine/management/test_compaction_settings.py @@ -36,10 +36,11 @@ def assert_option_fails(self, key): # key is a normal_key, converted to # __compaction_key__ - key = "__compaction_{}__".format(key) + key = "__compaction_{0}__".format(key) - with patch.object(self.model, key, 10), self.assertRaises(CQLEngineException): - get_compaction_options(self.model) + with patch.object(self.model, key, 10): + with self.assertRaises(CQLEngineException): + get_compaction_options(self.model) class SizeTieredCompactionTest(BaseCompactionTest): diff --git a/tests/integration/cqlengine/management/test_management.py b/tests/integration/cqlengine/management/test_management.py index 99b34c5cf0..1601f318c8 100644 --- a/tests/integration/cqlengine/management/test_management.py +++ b/tests/integration/cqlengine/management/test_management.py @@ -11,10 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa import mock - -from unittest import skipUnless import warnings from cassandra.cqlengine import CACHING_ALL, CACHING_NONE @@ -310,7 +312,7 @@ def test_failure(self): sync_table(self.FakeModel) -@skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") +@unittest.skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") def test_static_columns(): class StaticModel(Model): id = columns.Integer(primary_key=True) diff --git a/tests/integration/cqlengine/model/test_equality_operations.py b/tests/integration/cqlengine/model/test_equality_operations.py index 0c098deeac..8ed38327ef 100644 --- a/tests/integration/cqlengine/model/test_equality_operations.py +++ b/tests/integration/cqlengine/model/test_equality_operations.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest import skip from uuid import uuid4 from tests.integration.cqlengine.base import BaseCassEngTestCase diff --git a/tests/integration/cqlengine/model/test_model.py b/tests/integration/cqlengine/model/test_model.py index 0ac81c35e6..b66e284fae 100644 --- a/tests/integration/cqlengine/model/test_model.py +++ b/tests/integration/cqlengine/model/test_model.py @@ -11,15 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from unittest import TestCase +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa from cassandra.cqlengine import columns from cassandra.cqlengine.management import sync_table, drop_table, create_keyspace_simple, drop_keyspace from cassandra.cqlengine.models import Model, ModelDefinitionException -class TestModel(TestCase): +class TestModel(unittest.TestCase): """ Tests the non-io functionality of models """ def test_instance_equality(self): @@ -104,7 +106,7 @@ class table(Model): drop_keyspace('keyspace') -class BuiltInAttributeConflictTest(TestCase): +class BuiltInAttributeConflictTest(unittest.TestCase): """tests Model definitions that conflict with built-in attributes/methods""" def test_model_with_attribute_name_conflict(self): diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 2033d9dc92..8de0fe6ab0 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -11,10 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa from uuid import uuid4, UUID import random -import unittest from datetime import datetime, date, time from decimal import Decimal from operator import itemgetter diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 2d870d38ed..25c3ccee44 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -11,10 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa from datetime import datetime, date, time from decimal import Decimal -import unittest from uuid import UUID, uuid4 from cassandra.cqlengine.models import Model diff --git a/tests/integration/cqlengine/operators/test_base_operator.py b/tests/integration/cqlengine/operators/test_base_operator.py index 9a15f829fa..8df0da7f02 100644 --- a/tests/integration/cqlengine/operators/test_base_operator.py +++ b/tests/integration/cqlengine/operators/test_base_operator.py @@ -12,11 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest import TestCase +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + from cassandra.cqlengine.operators import BaseQueryOperator, QueryOperatorException -class BaseOperatorTest(TestCase): +class BaseOperatorTest(unittest.TestCase): def test_get_operator_cannot_be_called_from_base_class(self): with self.assertRaises(QueryOperatorException): diff --git a/tests/integration/cqlengine/query/test_batch_query.py b/tests/integration/cqlengine/query/test_batch_query.py index 1247f680b2..d9dc33c22d 100644 --- a/tests/integration/cqlengine/query/test_batch_query.py +++ b/tests/integration/cqlengine/query/test_batch_query.py @@ -116,7 +116,7 @@ def test_bulk_delete_success_case(self): for i in range(1): for j in range(5): - TestMultiKeyModel.create(partition=i, cluster=j, count=i*j, text='{}:{}'.format(i,j)) + TestMultiKeyModel.create(partition=i, cluster=j, count=i*j, text='{0}:{1}'.format(i,j)) with BatchQuery() as b: TestMultiKeyModel.objects.batch(b).filter(partition=0).delete() diff --git a/tests/integration/cqlengine/query/test_datetime_queries.py b/tests/integration/cqlengine/query/test_datetime_queries.py index 936f4fc78b..ebce19f449 100644 --- a/tests/integration/cqlengine/query/test_datetime_queries.py +++ b/tests/integration/cqlengine/query/test_datetime_queries.py @@ -14,6 +14,7 @@ from datetime import datetime, timedelta from uuid import uuid4 +from cassandra.cqlengine.functions import get_total_seconds from tests.integration.cqlengine.base import BaseCassEngTestCase @@ -66,6 +67,6 @@ def test_datetime_precision(self): obj = DateTimeQueryTestModel.create(user=pk, day=now, data='energy cheese') load = DateTimeQueryTestModel.get(user=pk) - assert abs(now - load.day).total_seconds() < 0.001 + self.assertAlmostEqual(get_total_seconds(now - load.day), 0, 2) obj.delete() diff --git a/tests/integration/cqlengine/query/test_queryoperators.py b/tests/integration/cqlengine/query/test_queryoperators.py index 4e28090672..f3154de90b 100644 --- a/tests/integration/cqlengine/query/test_queryoperators.py +++ b/tests/integration/cqlengine/query/test_queryoperators.py @@ -99,7 +99,7 @@ class TestModel(Model): q = TestModel.objects.filter(pk__token__gt=func) where = q._where[0] where.set_context_id(1) - self.assertEqual(str(where), 'token("p1", "p2") > token(%({})s, %({})s)'.format(1, 2)) + self.assertEqual(str(where), 'token("p1", "p2") > token(%({0})s, %({1})s)'.format(1, 2)) # Verify that a SELECT query can be successfully generated str(q._select_query()) @@ -111,7 +111,7 @@ class TestModel(Model): q = TestModel.objects.filter(pk__token__gt=func) where = q._where[0] where.set_context_id(1) - self.assertEqual(str(where), 'token("p1", "p2") > token(%({})s, %({})s)'.format(1, 2)) + self.assertEqual(str(where), 'token("p1", "p2") > token(%({0})s, %({1})s)'.format(1, 2)) str(q._select_query()) # The 'pk__token' virtual column may only be compared to a Token diff --git a/tests/integration/cqlengine/query/test_queryset.py b/tests/integration/cqlengine/query/test_queryset.py index 8d8610b16b..6d5fc926c5 100644 --- a/tests/integration/cqlengine/query/test_queryset.py +++ b/tests/integration/cqlengine/query/test_queryset.py @@ -11,11 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from __future__ import absolute_import + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + from datetime import datetime import time -from unittest import TestCase, skipUnless from uuid import uuid1, uuid4 import uuid @@ -685,7 +689,7 @@ def test_objects_property_returns_fresh_queryset(self): assert TestModel.objects._result_cache is None -@skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") +@unittest.skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") def test_paged_result_handling(): # addresses #225 class PagingTest(Model): diff --git a/tests/integration/cqlengine/query/test_updates.py b/tests/integration/cqlengine/query/test_updates.py index 799e74d135..a3b80f15f5 100644 --- a/tests/integration/cqlengine/query/test_updates.py +++ b/tests/integration/cqlengine/query/test_updates.py @@ -137,11 +137,11 @@ def test_set_add_updates(self): partition = uuid4() cluster = 1 TestQueryUpdateModel.objects.create( - partition=partition, cluster=cluster, text_set={"foo"}) + partition=partition, cluster=cluster, text_set=set(("foo",))) TestQueryUpdateModel.objects( - partition=partition, cluster=cluster).update(text_set__add={'bar'}) + partition=partition, cluster=cluster).update(text_set__add=set(('bar',))) obj = TestQueryUpdateModel.objects.get(partition=partition, cluster=cluster) - self.assertEqual(obj.text_set, {"foo", "bar"}) + self.assertEqual(obj.text_set, set(("foo", "bar"))) def test_set_add_updates_new_record(self): """ If the key doesn't exist yet, an update creates the record @@ -149,20 +149,20 @@ def test_set_add_updates_new_record(self): partition = uuid4() cluster = 1 TestQueryUpdateModel.objects( - partition=partition, cluster=cluster).update(text_set__add={'bar'}) + partition=partition, cluster=cluster).update(text_set__add=set(('bar',))) obj = TestQueryUpdateModel.objects.get(partition=partition, cluster=cluster) - self.assertEqual(obj.text_set, {"bar"}) + self.assertEqual(obj.text_set, set(("bar",))) def test_set_remove_updates(self): partition = uuid4() cluster = 1 TestQueryUpdateModel.objects.create( - partition=partition, cluster=cluster, text_set={"foo", "baz"}) + partition=partition, cluster=cluster, text_set=set(("foo", "baz"))) TestQueryUpdateModel.objects( partition=partition, cluster=cluster).update( - text_set__remove={'foo'}) + text_set__remove=set(('foo',))) obj = TestQueryUpdateModel.objects.get(partition=partition, cluster=cluster) - self.assertEqual(obj.text_set, {"baz"}) + self.assertEqual(obj.text_set, set(("baz",))) def test_set_remove_new_record(self): """ Removing something not in the set should silently do nothing @@ -170,12 +170,12 @@ def test_set_remove_new_record(self): partition = uuid4() cluster = 1 TestQueryUpdateModel.objects.create( - partition=partition, cluster=cluster, text_set={"foo"}) + partition=partition, cluster=cluster, text_set=set(("foo",))) TestQueryUpdateModel.objects( partition=partition, cluster=cluster).update( - text_set__remove={'afsd'}) + text_set__remove=set(('afsd',))) obj = TestQueryUpdateModel.objects.get(partition=partition, cluster=cluster) - self.assertEqual(obj.text_set, {"foo"}) + self.assertEqual(obj.text_set, set(("foo",))) def test_list_append_updates(self): partition = uuid4() diff --git a/tests/integration/cqlengine/statements/test_assignment_clauses.py b/tests/integration/cqlengine/statements/test_assignment_clauses.py index 76d526f225..9874fddcf0 100644 --- a/tests/integration/cqlengine/statements/test_assignment_clauses.py +++ b/tests/integration/cqlengine/statements/test_assignment_clauses.py @@ -11,12 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase from cassandra.cqlengine.statements import AssignmentClause, SetUpdateClause, ListUpdateClause, MapUpdateClause, MapDeleteClause, FieldDeleteClause, CounterUpdateClause -class AssignmentClauseTests(TestCase): +class AssignmentClauseTests(unittest.TestCase): def test_rendering(self): pass @@ -27,14 +30,14 @@ def test_insert_tuple(self): self.assertEqual(ac.insert_tuple(), ('a', 10)) -class SetUpdateClauseTests(TestCase): +class SetUpdateClauseTests(unittest.TestCase): def test_update_from_none(self): - c = SetUpdateClause('s', {1, 2}, previous=None) + c = SetUpdateClause('s', set((1, 2)), previous=None) c._analyze() c.set_context_id(0) - self.assertEqual(c._assignments, {1, 2}) + self.assertEqual(c._assignments, set((1, 2))) self.assertIsNone(c._additions) self.assertIsNone(c._removals) @@ -43,11 +46,11 @@ def test_update_from_none(self): ctx = {} c.update_context(ctx) - self.assertEqual(ctx, {'0': {1, 2}}) + self.assertEqual(ctx, {'0': set((1, 2))}) def test_null_update(self): """ tests setting a set to None creates an empty update statement """ - c = SetUpdateClause('s', None, previous={1, 2}) + c = SetUpdateClause('s', None, previous=set((1, 2))) c._analyze() c.set_context_id(0) @@ -64,7 +67,7 @@ def test_null_update(self): def test_no_update(self): """ tests an unchanged value creates an empty update statement """ - c = SetUpdateClause('s', {1, 2}, previous={1, 2}) + c = SetUpdateClause('s', set((1, 2)), previous=set((1, 2))) c._analyze() c.set_context_id(0) @@ -95,15 +98,15 @@ def test_update_empty_set(self): ctx = {} c.update_context(ctx) - self.assertEqual(ctx, {'0' : set()}) + self.assertEqual(ctx, {'0': set()}) def test_additions(self): - c = SetUpdateClause('s', {1, 2, 3}, previous={1, 2}) + c = SetUpdateClause('s', set((1, 2, 3)), previous=set((1, 2))) c._analyze() c.set_context_id(0) self.assertIsNone(c._assignments) - self.assertEqual(c._additions, {3}) + self.assertEqual(c._additions, set((3,))) self.assertIsNone(c._removals) self.assertEqual(c.get_context_size(), 1) @@ -111,42 +114,42 @@ def test_additions(self): ctx = {} c.update_context(ctx) - self.assertEqual(ctx, {'0': {3}}) + self.assertEqual(ctx, {'0': set((3,))}) def test_removals(self): - c = SetUpdateClause('s', {1, 2}, previous={1, 2, 3}) + c = SetUpdateClause('s', set((1, 2)), previous=set((1, 2, 3))) c._analyze() c.set_context_id(0) self.assertIsNone(c._assignments) self.assertIsNone(c._additions) - self.assertEqual(c._removals, {3}) + self.assertEqual(c._removals, set((3,))) self.assertEqual(c.get_context_size(), 1) self.assertEqual(str(c), '"s" = "s" - %(0)s') ctx = {} c.update_context(ctx) - self.assertEqual(ctx, {'0': {3}}) + self.assertEqual(ctx, {'0': set((3,))}) def test_additions_and_removals(self): - c = SetUpdateClause('s', {2, 3}, previous={1, 2}) + c = SetUpdateClause('s', set((2, 3)), previous=set((1, 2))) c._analyze() c.set_context_id(0) self.assertIsNone(c._assignments) - self.assertEqual(c._additions, {3}) - self.assertEqual(c._removals, {1}) + self.assertEqual(c._additions, set((3,))) + self.assertEqual(c._removals, set((1,))) self.assertEqual(c.get_context_size(), 2) self.assertEqual(str(c), '"s" = "s" + %(0)s, "s" = "s" - %(1)s') ctx = {} c.update_context(ctx) - self.assertEqual(ctx, {'0': {3}, '1': {1}}) + self.assertEqual(ctx, {'0': set((3,)), '1': set((1,))}) -class ListUpdateClauseTests(TestCase): +class ListUpdateClauseTests(unittest.TestCase): def test_update_from_none(self): c = ListUpdateClause('s', [1, 2, 3]) @@ -262,7 +265,7 @@ def test_shrinking_list_update(self): self.assertEqual(ctx, {'0': [1, 2, 3]}) -class MapUpdateTests(TestCase): +class MapUpdateTests(unittest.TestCase): def test_update(self): c = MapUpdateClause('s', {3: 0, 5: 6}, previous={5: 0, 3: 4}) @@ -298,7 +301,7 @@ def test_nulled_columns_arent_included(self): self.assertNotIn(1, c._updates) -class CounterUpdateTests(TestCase): +class CounterUpdateTests(unittest.TestCase): def test_positive_update(self): c = CounterUpdateClause('a', 5, 3) @@ -334,7 +337,7 @@ def noop_update(self): self.assertEqual(ctx, {'5': 0}) -class MapDeleteTests(TestCase): +class MapDeleteTests(unittest.TestCase): def test_update(self): c = MapDeleteClause('s', {3: 0}, {1: 2, 3: 4, 5: 6}) @@ -350,7 +353,7 @@ def test_update(self): self.assertEqual(ctx, {'0': 1, '1': 5}) -class FieldDeleteTests(TestCase): +class FieldDeleteTests(unittest.TestCase): def test_str(self): f = FieldDeleteClause("blake") diff --git a/tests/integration/cqlengine/statements/test_assignment_statement.py b/tests/integration/cqlengine/statements/test_assignment_statement.py index 40ba73be36..18f08dd2e8 100644 --- a/tests/integration/cqlengine/statements/test_assignment_statement.py +++ b/tests/integration/cqlengine/statements/test_assignment_statement.py @@ -11,12 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase from cassandra.cqlengine.statements import AssignmentStatement, StatementException -class AssignmentStatementTest(TestCase): +class AssignmentStatementTest(unittest.TestCase): def test_add_assignment_type_checking(self): """ tests that only assignment clauses can be added to queries """ diff --git a/tests/integration/cqlengine/statements/test_base_statement.py b/tests/integration/cqlengine/statements/test_base_statement.py index 5d8e6844a4..2d0f30e4c0 100644 --- a/tests/integration/cqlengine/statements/test_base_statement.py +++ b/tests/integration/cqlengine/statements/test_base_statement.py @@ -11,12 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase from cassandra.cqlengine.statements import BaseCQLStatement, StatementException -class BaseStatementTest(TestCase): +class BaseStatementTest(unittest.TestCase): def test_where_clause_type_checking(self): """ tests that only assignment clauses can be added to queries """ diff --git a/tests/integration/cqlengine/statements/test_insert_statement.py b/tests/integration/cqlengine/statements/test_insert_statement.py index 302e04d80b..75a2e71b86 100644 --- a/tests/integration/cqlengine/statements/test_insert_statement.py +++ b/tests/integration/cqlengine/statements/test_insert_statement.py @@ -11,13 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase from cassandra.cqlengine.statements import InsertStatement, StatementException, AssignmentClause import six -class InsertStatementTests(TestCase): +class InsertStatementTests(unittest.TestCase): def test_where_clause_failure(self): """ tests that where clauses cannot be added to Insert statements """ diff --git a/tests/integration/cqlengine/statements/test_select_statement.py b/tests/integration/cqlengine/statements/test_select_statement.py index b20feab774..4f920ce042 100644 --- a/tests/integration/cqlengine/statements/test_select_statement.py +++ b/tests/integration/cqlengine/statements/test_select_statement.py @@ -11,13 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase from cassandra.cqlengine.statements import SelectStatement, WhereClause from cassandra.cqlengine.operators import * import six -class SelectStatementTests(TestCase): +class SelectStatementTests(unittest.TestCase): def test_single_field_is_listified(self): """ tests that passing a string field into the constructor puts it into a list """ diff --git a/tests/integration/cqlengine/statements/test_update_statement.py b/tests/integration/cqlengine/statements/test_update_statement.py index 0f77d53ca4..6c7bea6dc5 100644 --- a/tests/integration/cqlengine/statements/test_update_statement.py +++ b/tests/integration/cqlengine/statements/test_update_statement.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase -from cassandra.cqlengine.columns import Set, List from cassandra.cqlengine.operators import * from cassandra.cqlengine.statements import (UpdateStatement, WhereClause, AssignmentClause, SetUpdateClause, @@ -21,7 +23,7 @@ import six -class UpdateStatementTests(TestCase): +class UpdateStatementTests(unittest.TestCase): def test_table_rendering(self): """ tests that fields are properly added to the select statement """ @@ -60,7 +62,7 @@ def test_additional_rendering(self): def test_update_set_add(self): us = UpdateStatement('table') - us.add_assignment_clause(SetUpdateClause('a', {1}, operation='add')) + us.add_assignment_clause(SetUpdateClause('a', set((1,)), operation='add')) self.assertEqual(six.text_type(us), 'UPDATE table SET "a" = "a" + %(0)s') def test_update_empty_set_add_does_not_assign(self): diff --git a/tests/integration/cqlengine/statements/test_where_clause.py b/tests/integration/cqlengine/statements/test_where_clause.py index d29bc690e2..542cb1f9f2 100644 --- a/tests/integration/cqlengine/statements/test_where_clause.py +++ b/tests/integration/cqlengine/statements/test_where_clause.py @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa -from unittest import TestCase import six from cassandra.cqlengine.operators import EqualsOperator from cassandra.cqlengine.statements import StatementException, WhereClause -class TestWhereClause(TestCase): +class TestWhereClause(unittest.TestCase): def test_operator_check(self): """ tests that creating a where statement with a non BaseWhereOperator object fails """ diff --git a/tests/integration/cqlengine/test_batch_query.py b/tests/integration/cqlengine/test_batch_query.py index 85f3019ac9..a83dd2e942 100644 --- a/tests/integration/cqlengine/test_batch_query.py +++ b/tests/integration/cqlengine/test_batch_query.py @@ -108,7 +108,7 @@ def test_bulk_delete_success_case(self): for i in range(1): for j in range(5): - TestMultiKeyModel.create(partition=i, cluster=j, count=i*j, text='{}:{}'.format(i,j)) + TestMultiKeyModel.create(partition=i, cluster=j, count=i*j, text='{0}:{1}'.format(i,j)) with BatchQuery() as b: TestMultiKeyModel.objects.batch(b).filter(partition=0).delete() diff --git a/tests/integration/cqlengine/test_ifnotexists.py b/tests/integration/cqlengine/test_ifnotexists.py index 98b192a5d7..6a4387351c 100644 --- a/tests/integration/cqlengine/test_ifnotexists.py +++ b/tests/integration/cqlengine/test_ifnotexists.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa import mock -from unittest import skipUnless from uuid import uuid4 from cassandra.cqlengine import columns @@ -71,7 +74,7 @@ def tearDownClass(cls): class IfNotExistsInsertTests(BaseIfNotExistsTest): - @skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") + @unittest.skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") def test_insert_if_not_exists_success(self): """ tests that insertion with if_not_exists work as expected """ @@ -103,7 +106,7 @@ def test_insert_if_not_exists_failure(self): self.assertEqual(tm.count, 9) self.assertEqual(tm.text, '111111111111') - @skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") + @unittest.skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") def test_batch_insert_if_not_exists_success(self): """ tests that batch insertion with if_not_exists work as expected """ diff --git a/tests/integration/cqlengine/test_load.py b/tests/integration/cqlengine/test_load.py index 8c8d1d7923..4c62a525f3 100644 --- a/tests/integration/cqlengine/test_load.py +++ b/tests/integration/cqlengine/test_load.py @@ -11,22 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa import gc import os import resource -from unittest import skipUnless from cassandra.cqlengine import columns from cassandra.cqlengine.models import Model from cassandra.cqlengine.management import sync_table + class LoadTest(Model): k = columns.Integer(primary_key=True) v = columns.Integer() -@skipUnless("LOADTEST" in os.environ, "LOADTEST not on") +@unittest.skipUnless("LOADTEST" in os.environ, "LOADTEST not on") def test_lots_of_queries(): sync_table(LoadTest) import objgraph diff --git a/tests/integration/cqlengine/test_timestamp.py b/tests/integration/cqlengine/test_timestamp.py index 5d37710ce6..dde4334ad6 100644 --- a/tests/integration/cqlengine/test_timestamp.py +++ b/tests/integration/cqlengine/test_timestamp.py @@ -40,8 +40,9 @@ def setUpClass(cls): class BatchTest(BaseTimestampTest): def test_batch_is_included(self): - with mock.patch.object(self.session, "execute") as m, BatchQuery(timestamp=timedelta(seconds=30)) as b: - TestTimestampModel.batch(b).create(count=1) + with mock.patch.object(self.session, "execute") as m: + with BatchQuery(timestamp=timedelta(seconds=30)) as b: + TestTimestampModel.batch(b).create(count=1) "USING TIMESTAMP".should.be.within(m.call_args[0][0].query_string) @@ -49,8 +50,9 @@ def test_batch_is_included(self): class CreateWithTimestampTest(BaseTimestampTest): def test_batch(self): - with mock.patch.object(self.session, "execute") as m, BatchQuery() as b: - TestTimestampModel.timestamp(timedelta(seconds=10)).batch(b).create(count=1) + with mock.patch.object(self.session, "execute") as m: + with BatchQuery() as b: + TestTimestampModel.timestamp(timedelta(seconds=10)).batch(b).create(count=1) query = m.call_args[0][0].query_string @@ -96,8 +98,9 @@ def test_instance_update_includes_timestamp_in_query(self): "USING TIMESTAMP".should.be.within(m.call_args[0][0].query_string) def test_instance_update_in_batch(self): - with mock.patch.object(self.session, "execute") as m, BatchQuery() as b: - self.instance.batch(b).timestamp(timedelta(seconds=30)).update(count=2) + with mock.patch.object(self.session, "execute") as m: + with BatchQuery() as b: + self.instance.batch(b).timestamp(timedelta(seconds=30)).update(count=2) query = m.call_args[0][0].query_string "USING TIMESTAMP".should.be.within(query) diff --git a/tests/integration/cqlengine/test_transaction.py b/tests/integration/cqlengine/test_transaction.py index bc64dc8452..eec6c631ef 100644 --- a/tests/integration/cqlengine/test_transaction.py +++ b/tests/integration/cqlengine/test_transaction.py @@ -11,10 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa import mock import six -from unittest import skipUnless from uuid import uuid4 from cassandra.cqlengine import columns @@ -32,7 +35,7 @@ class TestTransactionModel(Model): text = columns.Text(required=False) -@skipUnless(CASSANDRA_VERSION >= '2.0.0', "transactions only supported on cassandra 2.0 or higher") +@unittest.skipUnless(CASSANDRA_VERSION >= '2.0.0', "transactions only supported on cassandra 2.0 or higher") class TestTransaction(BaseCassEngTestCase): @classmethod diff --git a/tox.ini b/tox.ini index d694e7ab03..f608061da4 100644 --- a/tox.ini +++ b/tox.ini @@ -17,11 +17,6 @@ commands = {envpython} setup.py build_ext --inplace nosetests --verbosity=2 tests/unit/ nosetests --verbosity=2 tests/integration/cqlengine -[testenv:py26] -commands = {envpython} setup.py build_ext --inplace - nosetests --verbosity=2 tests/unit/ -# no cqlengine support for 2.6 right now - [testenv:pypy] deps = {[base]deps} commands = {envpython} setup.py build_ext --inplace From 09f258bdba2710a016cf134336adbac77f22d35b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 11 Jun 2015 17:17:58 -0500 Subject: [PATCH 0202/2431] Make tox only install unittest2 in python2.6 fixes some issues around test skipping caused by unittest2 in Python 2.7+ --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f608061da4..23b2aee1f4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,6 @@ envlist = py26,py27,pypy,py33,py34 [base] deps = nose mock - unittest2 PyYAML six @@ -17,6 +16,11 @@ commands = {envpython} setup.py build_ext --inplace nosetests --verbosity=2 tests/unit/ nosetests --verbosity=2 tests/integration/cqlengine +[testenv:py26] +deps = {[testenv]deps} + unittest2 +# test skipping is different in unittest2 for python 2.7+; let's just use it where needed + [testenv:pypy] deps = {[base]deps} commands = {envpython} setup.py build_ext --inplace From d3c9d10a4fa8b9fededd6c3f0917887f01b24655 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 11 Jun 2015 17:38:43 -0500 Subject: [PATCH 0203/2431] Make meta use CQL $$ quoting for Function body --- cassandra/metadata.py | 4 ++-- tests/integration/standard/test_metadata.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 843d96f38e..6f8200a8f4 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1118,14 +1118,14 @@ def as_cql_query(self, formatted=False): for n, t in zip(self.argument_names, self.type_signature)]) typ = self.return_type.cql_parameterized_type() lang = self.language - body = protect_value(self.body) + body = self.body on_null = "CALLED" if self.called_on_null_input else "RETURNS NULL" return "CREATE FUNCTION %(keyspace)s.%(name)s(%(arg_list)s)%(sep)s" \ "%(on_null)s ON NULL INPUT%(sep)s" \ "RETURNS %(typ)s%(sep)s" \ "LANGUAGE %(lang)s%(sep)s" \ - "AS %(body)s;" % locals() + "AS $$%(body)s$$;" % locals() @property def signature(self): diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 2bb88b50cf..cd7def5a97 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -1358,10 +1358,11 @@ def make_function_kwargs(self, called_on_null=True): def test_functions_after_udt(self): """ - Test to to ensure udt's come after functions in in keyspace dump + Test to to ensure functions come after UDTs in in keyspace dump test_functions_after_udt creates a basic function. Then queries that function and make sure that in the results that UDT's are listed before any corresponding functions, when we dump the keyspace + Ideally we would make a function that takes a udt type, but this presently fails because C* c059a56 requires udt to be frozen to create, but does not store meta indicating frozen SEE https://issues.apache.org/jira/browse/CASSANDRA-9186 From b941b1a2f56f3c694ed27d164949e530b60c37a8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 11 Jun 2015 17:42:50 -0500 Subject: [PATCH 0204/2431] cqle: don't depend on iter order for model attr access test --- tests/integration/cqlengine/model/test_model_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 8de0fe6ab0..574d072dc4 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -78,7 +78,7 @@ def test_model_read_as_dict(self): } self.assertEqual(sorted(tm.keys()), sorted(column_dict.keys())) - self.assertItemsEqual(tm.values(), column_dict.values()) + self.assertItemsEqual(sorted(tm.values()), sorted(column_dict.values())) self.assertEqual( sorted(tm.items(), key=itemgetter(0)), sorted(column_dict.items(), key=itemgetter(0))) From d101d3ce60a0abe77cccba3fc5cde42f640e23c7 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 12 Jun 2015 10:02:19 -0500 Subject: [PATCH 0205/2431] cqle test: improve value comparison for reading model values --- tests/integration/cqlengine/model/test_model_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 574d072dc4..35d7b81fee 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -78,7 +78,7 @@ def test_model_read_as_dict(self): } self.assertEqual(sorted(tm.keys()), sorted(column_dict.keys())) - self.assertItemsEqual(sorted(tm.values()), sorted(column_dict.values())) + self.assertSetEqual(set(tm.values()), set(column_dict.values())) self.assertEqual( sorted(tm.items(), key=itemgetter(0)), sorted(column_dict.items(), key=itemgetter(0))) From 5ea3f2242fc5af974d4fe119e04f05c4ff8e2ace Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 12 Jun 2015 10:28:00 -0500 Subject: [PATCH 0206/2431] doc: add note about not blocking in future callbacks --- cassandra/cluster.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 834872398b..1c9b6fabba 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -3305,6 +3305,11 @@ def add_callback(self, fn, *args, **kwargs): If the final result has already been seen when this method is called, the callback will be called immediately (before this method returns). + Note: in the case that the result is not available when the callback is added, + the callback is executed by IO event thread. This means that the callback + should not block or attempt further synchronous requests, because no further + IO will be processed until the callback returns. + **Important**: if the callback you attach results in an exception being raised, **the exception will be ignored**, so please ensure your callback handles all error cases that you care about. From f2403d0380cee0a0b670d26784b055c392aa6dca Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 12 Jun 2015 10:40:44 -0500 Subject: [PATCH 0207/2431] cqle doc: deprecated TimeUUID.from_datetime --- cassandra/cqlengine/columns.py | 7 ++++++- docs/api/cassandra/cqlengine/columns.rst | 2 ++ docs/cqlengine/upgrade_guide.rst | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cassandra/cqlengine/columns.py b/cassandra/cqlengine/columns.py index 8f8a6e2546..185cccfbbc 100644 --- a/cassandra/cqlengine/columns.py +++ b/cassandra/cqlengine/columns.py @@ -557,7 +557,12 @@ def from_datetime(self, dt): :param dt: datetime :type dt: datetime - :return: + :return: uuid1 + + .. deprecated:: 2.6.0 + + Use :func:`cassandra.util.uuid_from_time` + """ msg = "cqlengine.columns.TimeUUID.from_datetime is deprecated. Use cassandra.util.uuid_from_time instead." warnings.warn(msg, DeprecationWarning) diff --git a/docs/api/cassandra/cqlengine/columns.rst b/docs/api/cassandra/cqlengine/columns.rst index 7c4695d323..918ff65fa6 100644 --- a/docs/api/cassandra/cqlengine/columns.rst +++ b/docs/api/cassandra/cqlengine/columns.rst @@ -78,6 +78,8 @@ Columns of all types are initialized by passing :class:`.Column` attributes to t .. autoclass:: TimeUUID(**kwargs) + .. automethod:: from_datetime + .. autoclass:: TinyInt(**kwargs) .. autoclass:: UserDefinedType diff --git a/docs/cqlengine/upgrade_guide.rst b/docs/cqlengine/upgrade_guide.rst index 9af5876426..ee524cc7f8 100644 --- a/docs/cqlengine/upgrade_guide.rst +++ b/docs/cqlengine/upgrade_guide.rst @@ -150,3 +150,6 @@ After:: __discriminator_value__ = 'dog' +TimeUUID.from_datetime +---------------------- +This function is deprecated in favor of the core utility function :func:`~.uuid_from_time`. From fa1cb6809377f9f41643d4e44fb986ef94f2c692 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 12 Jun 2015 16:16:19 -0700 Subject: [PATCH 0208/2431] Windows testing support and stabilizations --- tests/integration/__init__.py | 117 +++++++++++------- tests/integration/cqlengine/__init__.py | 2 +- .../cqlengine/columns/test_validation.py | 2 +- .../integration/cqlengine/model/test_udts.py | 2 +- .../cqlengine/query/test_queryset.py | 7 +- tests/integration/long/test_consistency.py | 22 ++-- tests/integration/long/test_failure_types.py | 41 +++--- tests/integration/long/test_ipv6.py | 68 ++++++---- .../long/test_loadbalancingpolicies.py | 32 ++++- tests/integration/long/test_schema.py | 81 +++++------- tests/integration/long/utils.py | 12 +- tests/integration/standard/test_cluster.py | 14 +-- tests/integration/standard/test_concurrent.py | 56 ++++++--- tests/integration/standard/test_metadata.py | 36 ++---- tests/integration/standard/test_metrics.py | 10 +- tests/integration/standard/test_query.py | 6 + tests/integration/standard/test_types.py | 9 +- tests/integration/standard/test_udts.py | 10 +- .../cqlengine => stress_tests}/test_load.py | 40 +++--- .../test_multi_inserts.py | 20 ++- tests/unit/io/test_asyncorereactor.py | 8 +- tests/unit/io/test_libevreactor.py | 2 +- tests/unit/test_time_util.py | 4 +- 23 files changed, 347 insertions(+), 254 deletions(-) rename tests/{integration/cqlengine => stress_tests}/test_load.py (52%) rename tests/{integration/long => stress_tests}/test_multi_inserts.py (76%) mode change 100755 => 100644 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 1f43d6d652..3568b1ffc6 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -12,25 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time -import traceback - try: import unittest2 as unittest except ImportError: import unittest # noqa -import logging -log = logging.getLogger(__name__) - -import os +import os, six, time, sys, logging, traceback from threading import Event -import six from subprocess import call - from itertools import groupby +from cassandra import OperationTimedOut, ReadTimeout, ReadFailure, WriteTimeout, WriteFailure from cassandra.cluster import Cluster +from cassandra.protocol import ConfigurationException try: from ccmlib.cluster import Cluster as CCMCluster @@ -39,6 +33,8 @@ except ImportError as e: CCMClusterFactory = None +log = logging.getLogger(__name__) + CLUSTER_NAME = 'test_cluster' SINGLE_NODE_CLUSTER_NAME = 'single_node' MULTIDC_CLUSTER_NAME = 'multidc_test_cluster' @@ -84,7 +80,7 @@ def _tuple_version(version_string): USE_CASS_EXTERNAL = bool(os.getenv('USE_CASS_EXTERNAL', False)) -default_cassandra_version = '2.1.3' +default_cassandra_version = '2.1.5' if USE_CASS_EXTERNAL: if CCMClusterFactory: @@ -157,10 +153,21 @@ def remove_cluster(): global CCM_CLUSTER if CCM_CLUSTER: - log.debug("removing cluster %s", CCM_CLUSTER.name) - CCM_CLUSTER.remove() - CCM_CLUSTER = None - + log.debug("Removing cluster {0}".format(CCM_CLUSTER.name)) + tries = 0 + while tries < 100: + try: + CCM_CLUSTER.remove() + CCM_CLUSTER = None + return + except WindowsError: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 + time.sleep(1) + + raise RuntimeError("Failed to remove cluster after 100 attempts") def is_current_cluster(cluster_name, node_counts): global CCM_CLUSTER @@ -175,49 +182,55 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True): global CCM_CLUSTER if USE_CASS_EXTERNAL: if CCM_CLUSTER: - log.debug("Using external ccm cluster %s", CCM_CLUSTER.name) + log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name)) else: log.debug("Using unnamed external cluster") return if is_current_cluster(cluster_name, nodes): - log.debug("Using existing cluster %s", cluster_name) - return - - if CCM_CLUSTER: - log.debug("Stopping cluster %s", CCM_CLUSTER.name) - CCM_CLUSTER.stop() + log.debug("Using existing cluster, matching topology: {0}".format(cluster_name)) + else: + if CCM_CLUSTER: + log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name)) + CCM_CLUSTER.stop() - try: try: - cluster = CCMClusterFactory.load(path, cluster_name) - log.debug("Found existing ccm %s cluster; clearing", cluster_name) - cluster.clear() - cluster.set_install_dir(**CCM_KWARGS) + CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name) + log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name)) + CCM_CLUSTER.clear() + CCM_CLUSTER.set_install_dir(**CCM_KWARGS) except Exception: - log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS) - cluster = CCMCluster(path, cluster_name, **CCM_KWARGS) - cluster.set_configuration_options({'start_native_transport': True}) + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + + log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, CCM_KWARGS)) + CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS) + CCM_CLUSTER.set_configuration_options({'start_native_transport': True}) if CASSANDRA_VERSION >= '2.2': - cluster.set_configuration_options({'enable_user_defined_functions': True}) + CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True}) common.switch_cluster(path, cluster_name) - cluster.populate(nodes, ipformat=ipformat) - + CCM_CLUSTER.populate(nodes, ipformat=ipformat) + try: jvm_args = [] # This will enable the Mirroring query handler which will echo our custom payload k,v pairs back if PROTOCOL_VERSION >= 4: jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"] if start: - log.debug("Starting ccm %s cluster", cluster_name) - cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args) + log.debug("Starting CCM cluster: {0}".format(cluster_name)) + CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args) setup_keyspace(ipformat=ipformat) - - CCM_CLUSTER = cluster except Exception: - log.exception("Failed to start ccm cluster. Removing cluster.") + log.exception("Failed to start CCM cluster; removing cluster.") + + if os.name == "nt": + if CCM_CLUSTER: + for node in CCM_CLUSTER.nodes.itervalues(): + os.system("taskkill /F /PID " + str(node.pid)) + else: + call(["pkill", "-9", "-f", ".ccm"]) remove_cluster() - call(["pkill", "-9", "-f", ".ccm"]) raise @@ -240,6 +253,22 @@ def teardown_package(): log.warning('Did not find cluster: %s' % cluster_name) +def execute_until_pass(session, query): + tries = 0 + while tries < 100: + try: + return session.execute(query) + except ConfigurationException: + # keyspace/table was already created/dropped + return + except (OperationTimedOut, ReadTimeout, ReadFailure, WriteTimeout, WriteFailure): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 + + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(query)) + def setup_keyspace(ipformat=None): # wait for nodes to startup time.sleep(10) @@ -251,32 +280,32 @@ def setup_keyspace(ipformat=None): session = cluster.connect() try: - results = session.execute("SELECT keyspace_name FROM system.schema_keyspaces") + results = execute_until_pass(session, "SELECT keyspace_name FROM system.schema_keyspaces") existing_keyspaces = [row[0] for row in results] for ksname in ('test1rf', 'test2rf', 'test3rf'): if ksname in existing_keyspaces: - session.execute("DROP KEYSPACE %s" % ksname) + execute_until_pass(session, "DROP KEYSPACE %s" % ksname) ddl = ''' CREATE KEYSPACE test3rf WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}''' - session.execute(ddl) + execute_until_pass(session, ddl) ddl = ''' CREATE KEYSPACE test2rf WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '2'}''' - session.execute(ddl) + execute_until_pass(session, ddl) ddl = ''' CREATE KEYSPACE test1rf WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}''' - session.execute(ddl) + execute_until_pass(session, ddl) ddl = ''' CREATE TABLE test3rf.test ( k int PRIMARY KEY, v int )''' - session.execute(ddl) + execute_until_pass(session, ddl) except Exception: traceback.print_exc() diff --git a/tests/integration/cqlengine/__init__.py b/tests/integration/cqlengine/__init__.py index f0322c6bc8..c53884fcb2 100644 --- a/tests/integration/cqlengine/__init__.py +++ b/tests/integration/cqlengine/__init__.py @@ -28,7 +28,7 @@ def setup_package(): use_single_node() keyspace = 'cqlengine_test' - connection.setup(['localhost'], + connection.setup(['127.0.0.1'], protocol_version=PROTOCOL_VERSION, default_keyspace=keyspace) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index 5426548e5e..b645755574 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -358,7 +358,7 @@ class TestTimeUUIDFromDatetime(BaseCassEngTestCase): def test_conversion_specific_date(self): dt = datetime(1981, 7, 11, microsecond=555000) - uuid = TimeUUID.from_datetime(dt) + uuid = util.uuid_from_time(dt) from uuid import UUID assert isinstance(uuid, UUID) diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 25c3ccee44..86cc4596d8 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -26,7 +26,7 @@ from cassandra.cqlengine.management import sync_table, sync_type, create_keyspace_simple, drop_keyspace from cassandra.util import Date, Time -from tests.integration import get_server_versions, PROTOCOL_VERSION +from tests.integration import PROTOCOL_VERSION from tests.integration.cqlengine.base import BaseCassEngTestCase diff --git a/tests/integration/cqlengine/query/test_queryset.py b/tests/integration/cqlengine/query/test_queryset.py index 6d5fc926c5..cbaea21f2d 100644 --- a/tests/integration/cqlengine/query/test_queryset.py +++ b/tests/integration/cqlengine/query/test_queryset.py @@ -37,6 +37,7 @@ from cassandra.cqlengine import statements from cassandra.cqlengine import operators +from cassandra.util import uuid_from_time from cassandra.cqlengine.connection import get_session from tests.integration import PROTOCOL_VERSION @@ -582,17 +583,17 @@ def test_tzaware_datetime_support(self): TimeUUIDQueryModel.create( partition=pk, - time=columns.TimeUUID.from_datetime(midpoint_utc - timedelta(minutes=1)), + time=uuid_from_time(midpoint_utc - timedelta(minutes=1)), data='1') TimeUUIDQueryModel.create( partition=pk, - time=columns.TimeUUID.from_datetime(midpoint_utc), + time=uuid_from_time(midpoint_utc), data='2') TimeUUIDQueryModel.create( partition=pk, - time=columns.TimeUUID.from_datetime(midpoint_utc + timedelta(minutes=1)), + time=uuid_from_time(midpoint_utc + timedelta(minutes=1)), data='3') assert ['1', '2'] == [o.data for o in TimeUUIDQueryModel.filter( diff --git a/tests/integration/long/test_consistency.py b/tests/integration/long/test_consistency.py index db752cc4bd..1c860e9930 100644 --- a/tests/integration/long/test_consistency.py +++ b/tests/integration/long/test_consistency.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import struct, logging, sys, traceback, time +import struct, time, traceback, sys, logging from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, Unavailable from cassandra.cluster import Cluster from cassandra.policies import TokenAwarePolicy, RoundRobinPolicy, DowngradingConsistencyRetryPolicy from cassandra.query import SimpleStatement -from tests.integration import use_singledc, PROTOCOL_VERSION +from tests.integration import use_singledc, PROTOCOL_VERSION, execute_until_pass from tests.integration.long.utils import (force_stop, create_schema, wait_for_down, wait_for_up, start, CoordinatorStats) @@ -28,8 +28,6 @@ except ImportError: import unittest # noqa -log = logging.getLogger(__name__) - ALL_CONSISTENCY_LEVELS = set([ ConsistencyLevel.ANY, ConsistencyLevel.ONE, ConsistencyLevel.TWO, ConsistencyLevel.QUORUM, ConsistencyLevel.THREE, @@ -41,6 +39,8 @@ SINGLE_DC_CONSISTENCY_LEVELS = ALL_CONSISTENCY_LEVELS - MULTI_DC_CONSISTENCY_LEVELS +log = logging.getLogger(__name__) + def setup_module(): use_singledc() @@ -65,15 +65,7 @@ def _insert(self, session, keyspace, count, consistency_level=ConsistencyLevel.O for i in range(count): ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', consistency_level=consistency_level) - while True: - try: - session.execute(ss) - break - except (OperationTimedOut, WriteTimeout): - ex_type, ex, tb = sys.exc_info() - log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) - del tb - time.sleep(1) + execute_until_pass(session, ss) def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ONE): routing_key = struct.pack('>i', 0) @@ -81,7 +73,10 @@ def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ON ss = SimpleStatement('SELECT * FROM cf WHERE k = 0', consistency_level=consistency_level, routing_key=routing_key) + tries = 0 while True: + if tries > 100: + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(ss)) try: self.coordinator_stats.add_coordinator(session.execute_async(ss)) break @@ -89,6 +84,7 @@ def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ON ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb + tries += 1 time.sleep(1) def _assert_writes_succeed(self, session, keyspace, consistency_levels): diff --git a/tests/integration/long/test_failure_types.py b/tests/integration/long/test_failure_types.py index a927dc95f9..64d06f46a3 100644 --- a/tests/integration/long/test_failure_types.py +++ b/tests/integration/long/test_failure_types.py @@ -12,19 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys, logging, traceback + +from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, ReadFailure, WriteFailure,\ + FunctionFailure +from cassandra.cluster import Cluster +from cassandra.concurrent import execute_concurrent_with_args +from cassandra.query import SimpleStatement +from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace, remove_cluster try: import unittest2 as unittest except ImportError: import unittest - -from cassandra.cluster import Cluster -from cassandra import ConsistencyLevel -from cassandra import WriteFailure, ReadFailure, FunctionFailure -from cassandra.concurrent import execute_concurrent_with_args -from cassandra.query import SimpleStatement -from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace +log = logging.getLogger(__name__) def setup_module(): @@ -48,14 +50,10 @@ def setup_module(): def teardown_module(): """ The rest of the tests don't need custom tombstones - reset the config options so as to not mess with other tests. + remove the cluster so as to not interfere with other tests. """ if PROTOCOL_VERSION >= 4: - ccm_cluster = get_cluster() - config_options = {} - ccm_cluster.set_configuration_options(config_options) - if ccm_cluster is not None: - ccm_cluster.stop() + remove_cluster() class ClientExceptionTests(unittest.TestCase): @@ -83,6 +81,19 @@ def tearDown(self): # Restart the nodes to fully functional again self.setFailingNodes(failing_nodes, "testksfail") + def execute_concurrent_args_helper(self, session, query, params): + tries = 0 + while tries < 100: + try: + return execute_concurrent_with_args(session, query, params, concurrency=50) + except (ReadTimeout, WriteTimeout, OperationTimedOut, ReadFailure, WriteFailure): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 + + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(query)) + def setFailingNodes(self, failing_nodes, keyspace): """ This method will take in a set of failing nodes, and toggle all of the nodes in the provided list to fail @@ -210,11 +221,11 @@ def test_tombstone_overflow_read_failure(self): statement = self.session.prepare("INSERT INTO test3rf.test2 (k, v0,v1) VALUES (1,?,1)") parameters = [(x,) for x in range(3000)] - execute_concurrent_with_args(self.session, statement, parameters, concurrency=50) + self.execute_concurrent_args_helper(self.session, statement, parameters) statement = self.session.prepare("DELETE v1 FROM test3rf.test2 WHERE k = 1 AND v0 =?") parameters = [(x,) for x in range(2001)] - execute_concurrent_with_args(self.session, statement, parameters, concurrency=50) + self.execute_concurrent_args_helper(self.session, statement, parameters) self._perform_cql_statement( """ diff --git a/tests/integration/long/test_ipv6.py b/tests/integration/long/test_ipv6.py index 3f416561ad..e17f22aa07 100644 --- a/tests/integration/long/test_ipv6.py +++ b/tests/integration/long/test_ipv6.py @@ -12,27 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging -import os -import socket +import os, socket +from ccmlib import common from cassandra.cluster import Cluster, NoHostAvailable -from ccmlib import common -from tests.integration import use_cluster, remove_cluster, PROTOCOL_VERSION from cassandra.io.asyncorereactor import AsyncoreConnection -try: - from cassandra.io.libevreactor import LibevConnection -except ImportError: - LibevConnection = None +from tests import is_monkey_patched +from tests.integration import use_cluster, remove_cluster, PROTOCOL_VERSION +if is_monkey_patched(): + LibevConnection = -1 + AsyncoreConnection = -1 +else: + try: + from cassandra.io.libevreactor import LibevConnection + except ImportError: + LibevConnection = None try: import unittest2 as unittest except ImportError: import unittest # noqa -log = logging.getLogger(__name__) # If more modules do IPV6 testing, this can be moved down to integration.__init__. # For now, just keeping the clutter here @@ -40,12 +42,12 @@ def setup_module(module): - validate_ccm_viable() - validate_host_viable() - # We use a dedicated cluster (instead of common singledc, as in other tests) because - # it's most likely that the test host will only have one local ipv6 address (::1) - # singledc has three - use_cluster(IPV6_CLUSTER_NAME, [1], ipformat='::%d') + if os.name != "nt": + validate_host_viable() + # We use a dedicated cluster (instead of common singledc, as in other tests) because + # it's most likely that the test host will only have one local ipv6 address (::1) + # singledc has three + use_cluster(IPV6_CLUSTER_NAME, [1], ipformat='::%d') def teardown_module(): @@ -73,7 +75,8 @@ class IPV6ConnectionTest(object): connection_class = None def test_connect(self): - cluster = Cluster(connection_class=self.connection_class, contact_points=['::1'], protocol_version=PROTOCOL_VERSION) + cluster = Cluster(connection_class=self.connection_class, contact_points=['::1'], connect_timeout=10, + protocol_version=PROTOCOL_VERSION) session = cluster.connect() future = session.execute_async("SELECT * FROM system.local") future.result() @@ -81,26 +84,41 @@ def test_connect(self): cluster.shutdown() def test_error(self): - cluster = Cluster(connection_class=self.connection_class, contact_points=['::1'], port=9043, protocol_version=PROTOCOL_VERSION) - self.assertRaisesRegexp(NoHostAvailable, '\(\'Unable to connect.*%s.*::1\', 9043.*Connection refused.*' % os.errno.ECONNREFUSED, cluster.connect) + cluster = Cluster(connection_class=self.connection_class, contact_points=['::1'], port=9043, + connect_timeout=10, protocol_version=PROTOCOL_VERSION) + self.assertRaisesRegexp(NoHostAvailable, '\(\'Unable to connect.*%s.*::1\', 9043.*Connection refused.*' + % os.errno.ECONNREFUSED, cluster.connect) def test_error_multiple(self): if len(socket.getaddrinfo('localhost', 9043, socket.AF_UNSPEC, socket.SOCK_STREAM)) < 2: raise unittest.SkipTest('localhost only resolves one address') - cluster = Cluster(connection_class=self.connection_class, contact_points=['localhost'], port=9043, protocol_version=PROTOCOL_VERSION) - self.assertRaisesRegexp(NoHostAvailable, '\(\'Unable to connect.*Tried connecting to \[\(.*\(.*\].*Last error', cluster.connect) + cluster = Cluster(connection_class=self.connection_class, contact_points=['localhost'], port=9043, + connect_timeout=10, protocol_version=PROTOCOL_VERSION) + self.assertRaisesRegexp(NoHostAvailable, '\(\'Unable to connect.*Tried connecting to \[\(.*\(.*\].*Last error', + cluster.connect) class LibevConnectionTests(IPV6ConnectionTest, unittest.TestCase): connection_class = LibevConnection - @classmethod - def setup_class(cls): - if LibevConnection is None: - raise unittest.SkipTest('libev does not appear to be installed properly') + def setUp(self): + if os.name == "nt": + raise unittest.SkipTest("IPv6 is currently not supported under Windows") + + if LibevConnection == -1: + raise unittest.SkipTest("Can't test libev with monkey patching") + elif LibevConnection is None: + raise unittest.SkipTest("Libev does not appear to be installed properly") class AsyncoreConnectionTests(IPV6ConnectionTest, unittest.TestCase): connection_class = AsyncoreConnection + + def setUp(self): + if os.name == "nt": + raise unittest.SkipTest("IPv6 is currently not supported under Windows") + + if AsyncoreConnection == -1: + raise unittest.SkipTest("Can't test asyncore with monkey patching") \ No newline at end of file diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 7b2b87957d..465f9d763d 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -14,7 +14,8 @@ import struct, time, logging, sys, traceback -from cassandra import ConsistencyLevel, Unavailable, OperationTimedOut, ReadTimeout +from cassandra import ConsistencyLevel, Unavailable, OperationTimedOut, ReadTimeout, ReadFailure, \ + WriteTimeout, WriteFailure from cassandra.cluster import Cluster, NoHostAvailable from cassandra.concurrent import execute_concurrent_with_args from cassandra.metadata import murmur3 @@ -50,9 +51,20 @@ def teardown_class(cls): def _insert(self, session, keyspace, count=12, consistency_level=ConsistencyLevel.ONE): session.execute('USE %s' % keyspace) - ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', - consistency_level=consistency_level) - execute_concurrent_with_args(session, ss, [None] * count) + ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', consistency_level=consistency_level) + + tries = 0 + while tries < 100: + try: + execute_concurrent_with_args(session, ss, [None] * count) + return + except (OperationTimedOut, WriteTimeout, WriteFailure): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 + + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(ss)) def _query(self, session, keyspace, count=12, consistency_level=ConsistencyLevel.ONE, use_prepared=False): @@ -62,28 +74,36 @@ def _query(self, session, keyspace, count=12, self.prepared = session.prepare(query_string) for i in range(count): + tries = 0 while True: + if tries > 100: + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(self.prepared)) try: self.coordinator_stats.add_coordinator(session.execute_async(self.prepared.bind((0,)))) break - except (OperationTimedOut, ReadTimeout): + except (OperationTimedOut, ReadTimeout, ReadFailure): ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb + tries += 1 else: routing_key = struct.pack('>i', 0) for i in range(count): ss = SimpleStatement('SELECT * FROM %s.cf WHERE k = 0' % keyspace, consistency_level=consistency_level, routing_key=routing_key) + tries = 0 while True: + if tries > 100: + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(ss)) try: self.coordinator_stats.add_coordinator(session.execute_async(ss)) break - except (OperationTimedOut, ReadTimeout): + except (OperationTimedOut, ReadTimeout, ReadFailure): ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb + tries += 1 def test_token_aware_is_used_by_default(self): """ diff --git a/tests/integration/long/test_schema.py b/tests/integration/long/test_schema.py index 806726cf32..7da5203fd6 100644 --- a/tests/integration/long/test_schema.py +++ b/tests/integration/long/test_schema.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, sys, traceback +import logging -from cassandra import ConsistencyLevel, OperationTimedOut +from cassandra import ConsistencyLevel, AlreadyExists from cassandra.cluster import Cluster -from cassandra.protocol import ConfigurationException from cassandra.query import SimpleStatement -from tests.integration import use_singledc, PROTOCOL_VERSION + +from tests.integration import use_singledc, PROTOCOL_VERSION, execute_until_pass try: import unittest2 as unittest @@ -54,29 +54,29 @@ def test_recreates(self): for keyspace_number in range(5): keyspace = "ks_{0}".format(keyspace_number) - results = session.execute("SELECT keyspace_name FROM system.schema_keyspaces") + results = execute_until_pass(session, "SELECT keyspace_name FROM system.schema_keyspaces") existing_keyspaces = [row[0] for row in results] if keyspace in existing_keyspaces: drop = "DROP KEYSPACE {0}".format(keyspace) log.debug(drop) - session.execute(drop) + execute_until_pass(session, drop) create = "CREATE KEYSPACE {0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 3}}".format(keyspace) log.debug(create) - session.execute(create) + execute_until_pass(session, create) create = "CREATE TABLE {0}.cf (k int PRIMARY KEY, i int)".format(keyspace) log.debug(create) - session.execute(create) + execute_until_pass(session, create) use = "USE {0}".format(keyspace) log.debug(use) - session.execute(use) + execute_until_pass(session, use) insert = "INSERT INTO cf (k, i) VALUES (0, 0)" log.debug(insert) ss = SimpleStatement(insert, consistency_level=ConsistencyLevel.QUORUM) - session.execute(ss) + execute_until_pass(session, ss) def test_for_schema_disagreements_different_keyspaces(self): """ @@ -86,28 +86,13 @@ def test_for_schema_disagreements_different_keyspaces(self): session = self.session for i in xrange(30): - try: - session.execute("CREATE KEYSPACE test_{0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}".format(i)) - session.execute("CREATE TABLE test_{0}.cf (key int PRIMARY KEY, value int)".format(i)) - - for j in xrange(100): - session.execute("INSERT INTO test_{0}.cf (key, value) VALUES ({1}, {1})".format(i, j)) - except OperationTimedOut: - ex_type, ex, tb = sys.exc_info() - log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) - del tb - finally: - while True: - try: - session.execute("DROP KEYSPACE test_{0}".format(i)) - break - except OperationTimedOut: - ex_type, ex, tb = sys.exc_info() - log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) - del tb - except ConfigurationException: - # We're good, the keyspace was never created due to OperationTimedOut - break + execute_until_pass(session, "CREATE KEYSPACE test_{0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}".format(i)) + execute_until_pass(session, "CREATE TABLE test_{0}.cf (key int PRIMARY KEY, value int)".format(i)) + + for j in xrange(100): + execute_until_pass(session, "INSERT INTO test_{0}.cf (key, value) VALUES ({1}, {1})".format(i, j)) + + execute_until_pass(session, "DROP KEYSPACE test_{0}".format(i)) def test_for_schema_disagreements_same_keyspace(self): """ @@ -119,24 +104,14 @@ def test_for_schema_disagreements_same_keyspace(self): for i in xrange(30): try: - session.execute("CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}") - session.execute("CREATE TABLE test.cf (key int PRIMARY KEY, value int)") - - for j in xrange(100): - session.execute("INSERT INTO test.cf (key, value) VALUES ({0}, {0})".format(j)) - except OperationTimedOut: - ex_type, ex, tb = sys.exc_info() - log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) - del tb - finally: - while True: - try: - session.execute("DROP KEYSPACE test") - break - except OperationTimedOut: - ex_type, ex, tb = sys.exc_info() - log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) - del tb - except ConfigurationException: - # We're good, the keyspace was never created due to OperationTimedOut - break + execute_until_pass(session, "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}") + except AlreadyExists: + execute_until_pass(session, "DROP KEYSPACE test") + execute_until_pass(session, "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}") + + execute_until_pass(session, "CREATE TABLE test.cf (key int PRIMARY KEY, value int)") + + for j in xrange(100): + execute_until_pass(session, "INSERT INTO test.cf (key, value) VALUES ({0}, {0})".format(j)) + + execute_until_pass(session, "DROP KEYSPACE test") diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index e0e1e530d7..a3fb098cb5 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -131,23 +131,31 @@ def ring(node): def wait_for_up(cluster, node, wait=True): - while True: + tries = 0 + while tries < 100: host = cluster.metadata.get_host(IP_FORMAT % node) if host and host.is_up: log.debug("Done waiting for node %s to be up", node) return else: log.debug("Host is still marked down, waiting") + tries += 1 time.sleep(1) + raise RuntimeError("Host {0} is not up after 100 attempts".format(IP_FORMAT.format(node))) + def wait_for_down(cluster, node, wait=True): log.debug("Waiting for node %s to be down", node) - while True: + tries = 0 + while tries < 100: host = cluster.metadata.get_host(IP_FORMAT % node) if not host or not host.is_up: log.debug("Done waiting for node %s to be down", node) return else: log.debug("Host is still marked up, waiting") + tries += 1 time.sleep(1) + + raise RuntimeError("Host {0} is not down after 100 attempts".format(IP_FORMAT.format(node))) \ No newline at end of file diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index bf6bb1d476..045809a6f2 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -30,7 +30,7 @@ WhiteListRoundRobinPolicy) from cassandra.query import SimpleStatement, TraceUnavailable -from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions, get_node +from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions, get_node, execute_until_pass from tests.integration.util import assert_quiescent_pool_state @@ -72,14 +72,14 @@ def test_basic(self): cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() - result = session.execute( + result = execute_until_pass(session, """ CREATE KEYSPACE clustertests WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} """) self.assertEqual(None, result) - result = session.execute( + result = execute_until_pass(session, """ CREATE TABLE clustertests.cf0 ( a text, @@ -99,7 +99,7 @@ def test_basic(self): result = session.execute("SELECT * FROM clustertests.cf0") self.assertEqual([('a', 'b', 'c')], result) - session.execute("DROP KEYSPACE clustertests") + execute_until_pass(session, "DROP KEYSPACE clustertests") cluster.shutdown() @@ -227,7 +227,7 @@ def test_submit_schema_refresh(self): other_cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = other_cluster.connect() - session.execute( + execute_until_pass(session, """ CREATE KEYSPACE newkeyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} @@ -238,7 +238,7 @@ def test_submit_schema_refresh(self): self.assertIn("newkeyspace", cluster.metadata.keyspaces) - session.execute("DROP KEYSPACE newkeyspace") + execute_until_pass(session, "DROP KEYSPACE newkeyspace") cluster.shutdown() other_cluster.shutdown() @@ -303,7 +303,7 @@ def test_refresh_schema_type(self): keyspace_name = 'test1rf' type_name = self._testMethodName - session.execute('CREATE TYPE IF NOT EXISTS %s.%s (one int, two text)' % (keyspace_name, type_name)) + execute_until_pass(session, 'CREATE TYPE IF NOT EXISTS %s.%s (one int, two text)' % (keyspace_name, type_name)) original_meta = cluster.metadata.keyspaces original_test1rf_meta = original_meta[keyspace_name] original_type_meta = original_test1rf_meta.user_types[type_name] diff --git a/tests/integration/standard/test_concurrent.py b/tests/integration/standard/test_concurrent.py index 8b2bd8e696..9a82338348 100644 --- a/tests/integration/standard/test_concurrent.py +++ b/tests/integration/standard/test_concurrent.py @@ -12,6 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from itertools import cycle +import sys, logging, traceback + +from cassandra import InvalidRequest, ConsistencyLevel, ReadTimeout, WriteTimeout, OperationTimedOut, \ + ReadFailure, WriteFailure +from cassandra.cluster import Cluster, PagedResult +from cassandra.concurrent import execute_concurrent, execute_concurrent_with_args +from cassandra.policies import HostDistance +from cassandra.query import tuple_factory, SimpleStatement + from tests.integration import use_singledc, PROTOCOL_VERSION try: @@ -19,14 +29,7 @@ except ImportError: import unittest # noqa -from itertools import cycle - -from cassandra import InvalidRequest, ConsistencyLevel -from cassandra.cluster import Cluster, PagedResult -from cassandra.concurrent import (execute_concurrent, - execute_concurrent_with_args) -from cassandra.policies import HostDistance -from cassandra.query import tuple_factory, SimpleStatement +log = logging.getLogger(__name__) def setup_module(): @@ -47,6 +50,31 @@ def setUpClass(cls): def tearDownClass(cls): cls.cluster.shutdown() + def execute_concurrent_helper(self, session, query): + count = 0 + while count < 100: + try: + return execute_concurrent(session, query) + except (ReadTimeout, WriteTimeout, OperationTimedOut, ReadFailure, WriteFailure): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + count += 1 + + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(query)) + + def execute_concurrent_args_helper(self, session, query, params): + count = 0 + while count < 100: + try: + return execute_concurrent_with_args(session, query, params) + except (ReadTimeout, WriteTimeout, OperationTimedOut, ReadFailure, WriteFailure): + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(query)) + def test_execute_concurrent(self): for num_statements in (0, 1, 2, 7, 10, 99, 100, 101, 199, 200, 201): # write @@ -56,7 +84,7 @@ def test_execute_concurrent(self): statements = cycle((statement, )) parameters = [(i, i) for i in range(num_statements)] - results = execute_concurrent(self.session, list(zip(statements, parameters))) + results = self.execute_concurrent_helper(self.session, list(zip(statements, parameters))) self.assertEqual(num_statements, len(results)) self.assertEqual([(True, None)] * num_statements, results) @@ -67,7 +95,7 @@ def test_execute_concurrent(self): statements = cycle((statement, )) parameters = [(i, ) for i in range(num_statements)] - results = execute_concurrent(self.session, list(zip(statements, parameters))) + results = self.execute_concurrent_helper(self.session, list(zip(statements, parameters))) self.assertEqual(num_statements, len(results)) self.assertEqual([(True, [(i,)]) for i in range(num_statements)], results) @@ -78,7 +106,7 @@ def test_execute_concurrent_with_args(self): consistency_level=ConsistencyLevel.QUORUM) parameters = [(i, i) for i in range(num_statements)] - results = execute_concurrent_with_args(self.session, statement, parameters) + results = self.execute_concurrent_args_helper(self.session, statement, parameters) self.assertEqual(num_statements, len(results)) self.assertEqual([(True, None)] * num_statements, results) @@ -88,7 +116,7 @@ def test_execute_concurrent_with_args(self): consistency_level=ConsistencyLevel.QUORUM) parameters = [(i, ) for i in range(num_statements)] - results = execute_concurrent_with_args(self.session, statement, parameters) + results = self.execute_concurrent_args_helper(self.session, statement, parameters) self.assertEqual(num_statements, len(results)) self.assertEqual([(True, [(i,)]) for i in range(num_statements)], results) @@ -104,7 +132,7 @@ def test_execute_concurrent_paged_result(self): consistency_level=ConsistencyLevel.QUORUM) parameters = [(i, i) for i in range(num_statements)] - results = execute_concurrent_with_args(self.session, statement, parameters) + results = self.execute_concurrent_args_helper(self.session, statement, parameters) self.assertEqual(num_statements, len(results)) self.assertEqual([(True, None)] * num_statements, results) @@ -115,7 +143,7 @@ def test_execute_concurrent_paged_result(self): fetch_size=int(num_statements / 2)) parameters = [(i, ) for i in range(num_statements)] - results = execute_concurrent_with_args(self.session, statement, [(num_statements,)]) + results = self.execute_concurrent_args_helper(self.session, statement, [(num_statements,)]) self.assertEqual(1, len(results)) self.assertTrue(results[0][0]) result = results[0][1] diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 2bb88b50cf..ae1a37a09c 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -17,15 +17,10 @@ except ImportError: import unittest # noqa -import difflib +import difflib, six, sys from mock import Mock -import logging -import six -import sys -import traceback -from cassandra import AlreadyExists, OperationTimedOut, SignatureDescriptor, UserFunctionDescriptor, \ - UserAggregateDescriptor +from cassandra import AlreadyExists, SignatureDescriptor, UserFunctionDescriptor, UserAggregateDescriptor from cassandra.cluster import Cluster from cassandra.cqltypes import DoubleType, Int32Type, ListType, UTF8Type, MapType @@ -35,10 +30,7 @@ from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host -from tests.integration import (get_cluster, use_singledc, PROTOCOL_VERSION, - get_server_versions) - -log = logging.getLogger(__name__) +from tests.integration import get_cluster, use_singledc, PROTOCOL_VERSION, get_server_versions, execute_until_pass def setup_module(): @@ -58,18 +50,12 @@ def setUp(self): self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) self.session = self.cluster.connect() - self.session.execute("CREATE KEYSPACE schemametadatatest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}") + execute_until_pass(self.session, + "CREATE KEYSPACE schemametadatatest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}") def tearDown(self): - while True: - try: - self.session.execute("DROP KEYSPACE schemametadatatest") - self.cluster.shutdown() - break - except OperationTimedOut: - ex_type, ex, tb = sys.exc_info() - log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) - del tb + execute_until_pass(self.session, "DROP KEYSPACE schemametadatatest") + self.cluster.shutdown() def make_create_statement(self, partition_cols, clustering_cols=None, other_cols=None, compact=False): clustering_cols = clustering_cols or [] @@ -107,13 +93,13 @@ def make_create_statement(self, partition_cols, clustering_cols=None, other_cols def check_create_statement(self, tablemeta, original): recreate = tablemeta.as_cql_query(formatted=False) self.assertEqual(original, recreate[:len(original)]) - self.session.execute("DROP TABLE %s.%s" % (self.ksname, self.cfname)) - self.session.execute(recreate) + execute_until_pass(self.session, "DROP TABLE {0}.{1}".format(self.ksname, self.cfname)) + execute_until_pass(self.session, recreate) # create the table again, but with formatting enabled - self.session.execute("DROP TABLE %s.%s" % (self.ksname, self.cfname)) + execute_until_pass(self.session, "DROP TABLE {0}.{1}".format(self.ksname, self.cfname)) recreate = tablemeta.as_cql_query(formatted=True) - self.session.execute(recreate) + execute_until_pass(self.session, recreate) def get_table_metadata(self): self.cluster.refresh_table_metadata(self.ksname, self.cfname) diff --git a/tests/integration/standard/test_metrics.py b/tests/integration/standard/test_metrics.py index 7b19404907..6731e9073e 100644 --- a/tests/integration/standard/test_metrics.py +++ b/tests/integration/standard/test_metrics.py @@ -23,7 +23,7 @@ from cassandra import ConsistencyLevel, WriteTimeout, Unavailable, ReadTimeout from cassandra.cluster import Cluster, NoHostAvailable -from tests.integration import get_cluster, get_node, use_singledc, PROTOCOL_VERSION +from tests.integration import get_cluster, get_node, use_singledc, PROTOCOL_VERSION, execute_until_pass def setup_module(): @@ -76,7 +76,7 @@ def test_write_timeout(self): # Assert read query = SimpleStatement("SELECT * FROM test WHERE k=1", consistency_level=ConsistencyLevel.ALL) - results = session.execute(query) + results = execute_until_pass(session, query) self.assertEqual(1, len(results)) # Pause node so it shows as unreachable to coordinator @@ -109,7 +109,7 @@ def test_read_timeout(self): # Assert read query = SimpleStatement("SELECT * FROM test WHERE k=1", consistency_level=ConsistencyLevel.ALL) - results = session.execute(query) + results = execute_until_pass(session, query) self.assertEqual(1, len(results)) # Pause node so it shows as unreachable to coordinator @@ -142,11 +142,11 @@ def test_unavailable(self): # Assert read query = SimpleStatement("SELECT * FROM test WHERE k=1", consistency_level=ConsistencyLevel.ALL) - results = session.execute(query) + results = execute_until_pass(session, query) self.assertEqual(1, len(results)) # Stop node gracefully - get_node(1).stop(wait=True, gently=True) + get_node(1).stop(wait=True, wait_other_notice=True) try: # Test write diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index dc98505b87..80a0d8e24f 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -512,8 +512,14 @@ def test_no_connection_refused_on_timeout(self): if type(result).__name__ == "WriteTimeout": received_timeout = True continue + if type(result).__name__ == "WriteFailure": + received_timeout = True + continue if type(result).__name__ == "ReadTimeout": continue + if type(result).__name__ == "ReadFailure": + continue + self.fail("Unexpected exception %s: %s" % (type(result).__name__, result.message)) # Make sure test passed diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index c6c8114b57..3dd87e82b9 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -17,9 +17,6 @@ except ImportError: import unittest # noqa -import logging -log = logging.getLogger(__name__) - from datetime import datetime import six @@ -29,7 +26,7 @@ from cassandra.query import dict_factory, ordered_dict_factory from cassandra.util import sortedset -from tests.integration import get_server_versions, use_singledc, PROTOCOL_VERSION +from tests.integration import get_server_versions, use_singledc, PROTOCOL_VERSION, execute_until_pass from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, COLLECTION_TYPES, \ get_sample, get_collection_sample @@ -51,9 +48,9 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - cls.session.execute("DROP KEYSPACE typetests") + execute_until_pass(cls.session, "DROP KEYSPACE typetests") cls.cluster.shutdown() - + def test_can_insert_blob_type_as_string(self): """ Tests that byte strings in Python maps to blob type in Cassandra diff --git a/tests/integration/standard/test_udts.py b/tests/integration/standard/test_udts.py index 10b9dd390c..7d9eb99173 100644 --- a/tests/integration/standard/test_udts.py +++ b/tests/integration/standard/test_udts.py @@ -17,9 +17,6 @@ except ImportError: import unittest # noqa -import logging -log = logging.getLogger(__name__) - from collections import namedtuple from functools import partial @@ -28,7 +25,7 @@ from cassandra.query import dict_factory from cassandra.util import OrderedMap -from tests.integration import get_server_versions, use_singledc, PROTOCOL_VERSION +from tests.integration import get_server_versions, use_singledc, PROTOCOL_VERSION, execute_until_pass from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, COLLECTION_TYPES, \ get_sample, get_collection_sample @@ -51,13 +48,14 @@ def setUp(self): self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) self.session = self.cluster.connect() - self.session.execute("CREATE KEYSPACE udttests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") + execute_until_pass(self.session, + "CREATE KEYSPACE udttests WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") self.cluster.shutdown() def tearDown(self): self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) self.session = self.cluster.connect() - self.session.execute("DROP KEYSPACE udttests") + execute_until_pass(self.session, "DROP KEYSPACE udttests") self.cluster.shutdown() def test_can_insert_unprepared_registered_udts(self): diff --git a/tests/integration/cqlengine/test_load.py b/tests/stress_tests/test_load.py similarity index 52% rename from tests/integration/cqlengine/test_load.py rename to tests/stress_tests/test_load.py index 4c62a525f3..6283d2e413 100644 --- a/tests/integration/cqlengine/test_load.py +++ b/tests/stress_tests/test_load.py @@ -17,35 +17,37 @@ import unittest # noqa import gc -import os -import resource from cassandra.cqlengine import columns from cassandra.cqlengine.models import Model from cassandra.cqlengine.management import sync_table +from tests.integration.cqlengine.base import BaseCassEngTestCase -class LoadTest(Model): - k = columns.Integer(primary_key=True) - v = columns.Integer() +class LoadTests(BaseCassEngTestCase): -@unittest.skipUnless("LOADTEST" in os.environ, "LOADTEST not on") -def test_lots_of_queries(): - sync_table(LoadTest) - import objgraph - gc.collect() - objgraph.show_most_common_types() + def test_lots_of_queries(self): + import resource + import objgraph - print("Starting...") + class LoadTest(Model): + k = columns.Integer(primary_key=True) + v = columns.Integer() - for i in range(1000000): - if i % 25000 == 0: - # print memory statistic - print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)) + sync_table(LoadTest) + gc.collect() + objgraph.show_most_common_types() - LoadTest.create(k=i, v=i) + print("Starting...") - objgraph.show_most_common_types() + for i in range(1000000): + if i % 25000 == 0: + # print memory statistic + print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)) - raise Exception("you shouldn't be here") + LoadTest.create(k=i, v=i) + + objgraph.show_most_common_types() + + raise Exception("you shouldn't be here") diff --git a/tests/integration/long/test_multi_inserts.py b/tests/stress_tests/test_multi_inserts.py old mode 100755 new mode 100644 similarity index 76% rename from tests/integration/long/test_multi_inserts.py rename to tests/stress_tests/test_multi_inserts.py index 10422c974b..b23a29ddca --- a/tests/integration/long/test_multi_inserts.py +++ b/tests/stress_tests/test_multi_inserts.py @@ -1,3 +1,16 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. try: import unittest2 as unittest @@ -14,11 +27,10 @@ def setup_module(): class StressInsertsTests(unittest.TestCase): - - ''' + """ Test case for PYTHON-124: Repeated inserts may exhaust all connections causing NoConnectionsAvailable, in_flight never decreased - ''' + """ def setUp(self): """ @@ -61,7 +73,7 @@ def test_in_flight_is_one(self): for pool in self.session._pools.values(): if leaking_connections: break - for conn in pool._connections: + for conn in pool.get_connections(): if conn.in_flight > 1: print self.session.get_pool_state() leaking_connections = True diff --git a/tests/unit/io/test_asyncorereactor.py b/tests/unit/io/test_asyncorereactor.py index 17e42a0221..988f648869 100644 --- a/tests/unit/io/test_asyncorereactor.py +++ b/tests/unit/io/test_asyncorereactor.py @@ -44,7 +44,7 @@ class AsyncoreConnectionTest(unittest.TestCase): @classmethod def setUpClass(cls): if is_monkey_patched(): - raise unittest.SkipTest("monkey-patching detected") + return AsyncoreConnection.initialize_reactor() cls.socket_patcher = patch('socket.socket', spec=socket.socket) cls.mock_socket = cls.socket_patcher.start() @@ -56,8 +56,14 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + if is_monkey_patched(): + return cls.socket_patcher.stop() + def setUp(self): + if is_monkey_patched(): + raise unittest.SkipTest("Can't test asyncore with monkey patching") + def make_connection(self): c = AsyncoreConnection('1.2.3.4', cql_version='3.0.1') c.socket = Mock() diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index ddd2dc0417..39d66b6813 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -49,7 +49,7 @@ class LibevConnectionTest(unittest.TestCase): def setUp(self): if 'gevent.monkey' in sys.modules: - raise unittest.SkipTest("gevent monkey-patching detected") + raise unittest.SkipTest("Can't test libev with monkey patching") if LibevConnection is None: raise unittest.SkipTest('libev does not appear to be installed correctly') LibevConnection.initialize_reactor() diff --git a/tests/unit/test_time_util.py b/tests/unit/test_time_util.py index 0a3209a5f9..1c3dc1b4d1 100644 --- a/tests/unit/test_time_util.py +++ b/tests/unit/test_time_util.py @@ -42,11 +42,11 @@ def test_times_from_uuid1(self): u = uuid.uuid1(node, 0) t = util.unix_time_from_uuid1(u) - self.assertAlmostEqual(now, t, 3) + self.assertAlmostEqual(now, t, 2) dt = util.datetime_from_uuid1(u) t = calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6 - self.assertAlmostEqual(now, t, 3) + self.assertAlmostEqual(now, t, 2) def test_uuid_from_time(self): t = time.time() From b4a13d4bf0f34702fdd721f1a4bb27e4079e172c Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Fri, 12 Jun 2015 17:24:55 -0700 Subject: [PATCH 0209/2431] Cqlengine fixes to skip tests properly --- .../cqlengine/columns/test_validation.py | 9 ++++- .../cqlengine/management/test_management.py | 40 ++++++++++--------- .../cqlengine/model/test_model_io.py | 9 ++++- .../integration/cqlengine/model/test_udts.py | 3 +- .../cqlengine/query/test_queryset.py | 28 +++++++------ 5 files changed, 52 insertions(+), 37 deletions(-) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index b645755574..ce8ab987c5 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -153,14 +153,19 @@ class DateTest(Model): @classmethod def setUpClass(cls): if PROTOCOL_VERSION < 4: - raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) - + return sync_table(cls.DateTest) @classmethod def tearDownClass(cls): + if PROTOCOL_VERSION < 4: + return drop_table(cls.DateTest) + def setUp(self): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + def test_date_io(self): today = date.today() self.DateTest.objects.create(test_id=0, created_at=today) diff --git a/tests/integration/cqlengine/management/test_management.py b/tests/integration/cqlengine/management/test_management.py index 1601f318c8..e20680d036 100644 --- a/tests/integration/cqlengine/management/test_management.py +++ b/tests/integration/cqlengine/management/test_management.py @@ -311,30 +311,32 @@ def test_failure(self): with self.assertRaises(CQLEngineException): sync_table(self.FakeModel) +class StaticColumnTests(BaseCassEngTestCase): + def test_static_columns(self): + if PROTOCOL_VERSION < 2: + raise unittest.SkipTest("Native protocol 2+ required, currently using: {0}".format(PROTOCOL_VERSION)) -@unittest.skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") -def test_static_columns(): - class StaticModel(Model): - id = columns.Integer(primary_key=True) - c = columns.Integer(primary_key=True) - name = columns.Text(static=True) + class StaticModel(Model): + id = columns.Integer(primary_key=True) + c = columns.Integer(primary_key=True) + name = columns.Text(static=True) - drop_table(StaticModel) + drop_table(StaticModel) - session = get_session() + session = get_session() - with mock.patch.object(session, "execute", wraps=session.execute) as m: - sync_table(StaticModel) - - assert m.call_count > 0 - statement = m.call_args[0][0].query_string - assert '"name" text static' in statement, statement + with mock.patch.object(session, "execute", wraps=session.execute) as m: + sync_table(StaticModel) - # if we sync again, we should not apply an alter w/ a static - sync_table(StaticModel) + assert m.call_count > 0 + statement = m.call_args[0][0].query_string + assert '"name" text static' in statement, statement - with mock.patch.object(session, "execute", wraps=session.execute) as m2: + # if we sync again, we should not apply an alter w/ a static sync_table(StaticModel) - assert len(m2.call_args_list) == 1 - assert "ALTER" not in m2.call_args[0][0].query_string + with mock.patch.object(session, "execute", wraps=session.execute) as m2: + sync_table(StaticModel) + + assert len(m2.call_args_list) == 1 + assert "ALTER" not in m2.call_args[0][0].query_string diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 35d7b81fee..3ba2aa9711 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -437,7 +437,7 @@ class TestQuerying(BaseCassEngTestCase): @classmethod def setUpClass(cls): if PROTOCOL_VERSION < 4: - raise unittest.SkipTest("Date query tests require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + return super(TestQuerying, cls).setUpClass() drop_table(TestQueryModel) @@ -445,9 +445,16 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + if PROTOCOL_VERSION < 4: + return + super(TestQuerying, cls).tearDownClass() drop_table(TestQueryModel) + def setUp(self): + if PROTOCOL_VERSION < 4: + raise unittest.SkipTest("Date query tests require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) + def test_query_with_date(self): uid = uuid4() day = date(2013, 11, 26) diff --git a/tests/integration/cqlengine/model/test_udts.py b/tests/integration/cqlengine/model/test_udts.py index 86cc4596d8..bf2f37020c 100644 --- a/tests/integration/cqlengine/model/test_udts.py +++ b/tests/integration/cqlengine/model/test_udts.py @@ -32,8 +32,7 @@ class UserDefinedTypeTests(BaseCassEngTestCase): - @classmethod - def setUpClass(self): + def setUp(self): if PROTOCOL_VERSION < 3: raise unittest.SkipTest("UDTs require native protocol 3+, currently using: {0}".format(PROTOCOL_VERSION)) diff --git a/tests/integration/cqlengine/query/test_queryset.py b/tests/integration/cqlengine/query/test_queryset.py index cbaea21f2d..5c2b76f985 100644 --- a/tests/integration/cqlengine/query/test_queryset.py +++ b/tests/integration/cqlengine/query/test_queryset.py @@ -689,23 +689,25 @@ def test_objects_property_returns_fresh_queryset(self): len(TestModel.objects) # evaluate queryset assert TestModel.objects._result_cache is None +class PageQueryTests(BaseCassEngTestCase): + def test_paged_result_handling(self): + if PROTOCOL_VERSION < 2: + raise unittest.SkipTest("Paging requires native protocol 2+, currently using: {0}".format(PROTOCOL_VERSION)) -@unittest.skipUnless(PROTOCOL_VERSION >= 2, "only runs against the cql3 protocol v2.0") -def test_paged_result_handling(): - # addresses #225 - class PagingTest(Model): - id = columns.Integer(primary_key=True) - val = columns.Integer() - sync_table(PagingTest) + # addresses #225 + class PagingTest(Model): + id = columns.Integer(primary_key=True) + val = columns.Integer() + sync_table(PagingTest) - PagingTest.create(id=1, val=1) - PagingTest.create(id=2, val=2) + PagingTest.create(id=1, val=1) + PagingTest.create(id=2, val=2) - session = get_session() - with mock.patch.object(session, 'default_fetch_size', 1): - results = PagingTest.objects()[:] + session = get_session() + with mock.patch.object(session, 'default_fetch_size', 1): + results = PagingTest.objects()[:] - assert len(results) == 2 + assert len(results) == 2 class ModelQuerySetTimeoutTestCase(BaseQuerySetUsage): From b3027786239ec1c397aa7b4e06a5b157f8f07f44 Mon Sep 17 00:00:00 2001 From: Mike Okner Date: Fri, 12 Jun 2015 14:30:47 -0500 Subject: [PATCH 0210/2431] Set models.DEFAULT_KEYSPACE when calling set_session() Sets the default keyspace for cqlengine models when passing in an existing session to cqlengine.connection.set_session() if the session's keyspace is set and models.DEFAULT_KEYSPACE isn't. Without this, the user will see an error when trying to perform Cassandra interactions with a Model instance like "Keyspace none does not exist" unless they explicitly set models.DEFAULT_KEYSPACE themselves ahead of time or have a __keyspace__ attribute set on the model. Also no longer raising an exception in cqlengine.connection.setup() if default_keyspace isn't set since individual models might still have __keyspace__ set. --- cassandra/cqlengine/connection.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cassandra/cqlengine/connection.py b/cassandra/cqlengine/connection.py index ead241c6fd..c6540d4986 100644 --- a/cassandra/cqlengine/connection.py +++ b/cassandra/cqlengine/connection.py @@ -82,6 +82,11 @@ def set_session(s): session = s cluster = s.cluster + # Set default keyspace from given session's keyspace if not already set + from cassandra.cqlengine import models + if not models.DEFAULT_KEYSPACE and session.keyspace: + models.DEFAULT_KEYSPACE = session.keyspace + _register_known_types(cluster) log.debug("cqlengine connection initialized with %s", s) @@ -109,9 +114,6 @@ def setup( if 'username' in kwargs or 'password' in kwargs: raise CQLEngineException("Username & Password are now handled by using the native driver's auth_provider") - if not default_keyspace: - raise UndefinedKeyspaceException() - from cassandra.cqlengine import models models.DEFAULT_KEYSPACE = default_keyspace From be6337ee82fa30a53cd873e88d07838a8a3d0932 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 15 Jun 2015 15:32:53 -0500 Subject: [PATCH 0211/2431] Do not try to access an empty token map. Fixes an issue where multiple queries, with routing key set, to a keyspace without tokens_to_hosts, would raise a key error. --- cassandra/metadata.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 843d96f38e..857542c472 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -538,7 +538,6 @@ def add_or_return_host(self, host): self._hosts[host.address] = host return host, True - def remove_host(self, host): with self._hosts_lock: return bool(self._hosts.pop(host.address, False)) @@ -894,6 +893,7 @@ def _drop_table_metadata(self, table_name): for index_name in table_meta.indexes: self.indexes.pop(index_name, None) + class UserType(object): """ A user defined type, as created by ``CREATE TYPE`` statements. @@ -1607,17 +1607,17 @@ def get_replicas(self, keyspace, token): if tokens_to_hosts is None: self.rebuild_keyspace(keyspace, build_if_absent=True) tokens_to_hosts = self.tokens_to_hosts_by_ks.get(keyspace, None) - if not tokens_to_hosts: - return [] - - # token range ownership is exclusive on the LHS (the start token), so - # we use bisect_right, which, in the case of a tie/exact match, - # picks an insertion point to the right of the existing match - point = bisect_right(self.ring, token) - if point == len(self.ring): - return tokens_to_hosts[self.ring[0]] - else: - return tokens_to_hosts[self.ring[point]] + + if tokens_to_hosts: + # token range ownership is exclusive on the LHS (the start token), so + # we use bisect_right, which, in the case of a tie/exact match, + # picks an insertion point to the right of the existing match + point = bisect_right(self.ring, token) + if point == len(self.ring): + return tokens_to_hosts[self.ring[0]] + else: + return tokens_to_hosts[self.ring[point]] + return [] class Token(object): From beb14443d4688cc2c126afe91c193d127f5d1eed Mon Sep 17 00:00:00 2001 From: GregBestland Date: Mon, 15 Jun 2015 18:19:58 -0500 Subject: [PATCH 0212/2431] Adding integration tests for protocol negotiation --- tests/integration/standard/test_cluster.py | 38 +++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index bf6bb1d476..cbd8cd0996 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -30,7 +30,9 @@ WhiteListRoundRobinPolicy) from cassandra.query import SimpleStatement, TraceUnavailable -from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions, get_node + +from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions, get_node, CASSANDRA_VERSION + from tests.integration.util import assert_quiescent_pool_state @@ -103,6 +105,38 @@ def test_basic(self): cluster.shutdown() + def test_protocol_negotiation(self): + """ + Test for protocol negotiation + + test_protocol_negotiation tests that the driver will select the correct protocol version to match + the correct cassandra version. Please note that 2.1.5 has a + bug https://issues.apache.org/jira/browse/CASSANDRA-9451 that will cause this test to fail + that will cause this to not pass. It was rectified in 2.1.6 + + @since 2.6.0 + @jira_ticket PYTHON-240 + @expected_result the correct protocol version should be selected + + @test_category connection + """ + + cluster = Cluster() + session = cluster.connect() + default_protocol_version = session._protocol_version + + # Make sure the correct protocol was selected by default + if CASSANDRA_VERSION >= '2.2': + self.assertEqual(default_protocol_version, 4) + elif CASSANDRA_VERSION >= '2.1': + self.assertEqual(default_protocol_version, 3) + elif CASSANDRA_VERSION >= '2.0': + self.assertEqual(default_protocol_version, 2) + else: + self.assertEqual(default_protocol_version, 1) + + cluster.shutdown() + def test_connect_on_keyspace(self): """ Ensure clusters that connect on a keyspace, do @@ -577,3 +611,5 @@ def test_pool_management(self): assert_quiescent_pool_state(self, cluster) cluster.shutdown() + + From 2f3e0424698c1e8593ba3ef44169d68bd16e6038 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Tue, 16 Jun 2015 15:48:59 -0500 Subject: [PATCH 0213/2431] Adding basic integration test for PYTHON-322 --- tests/integration/long/ssl/cassandra.crt | Bin 0 -> 626 bytes tests/integration/long/ssl/driver_ca_cert.pem | 16 +++ tests/integration/long/ssl/keystore.jks | Bin 0 -> 1398 bytes tests/integration/long/test_ssl.py | 111 ++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 tests/integration/long/ssl/cassandra.crt create mode 100644 tests/integration/long/ssl/driver_ca_cert.pem create mode 100644 tests/integration/long/ssl/keystore.jks create mode 100644 tests/integration/long/test_ssl.py diff --git a/tests/integration/long/ssl/cassandra.crt b/tests/integration/long/ssl/cassandra.crt new file mode 100644 index 0000000000000000000000000000000000000000..432e58540babe586367427cfa5f5d7fdf8c2f170 GIT binary patch literal 626 zcmXqLV#+gUV!Xb9nTe5!iN*eN$qNHsHcqWJkGAi;jEvl@3le43th=CADhFzFDI5DpzQNcMUu_)0{(10H#$|cO~9<7#bKtxpZ|)<9y_h zU}R-rZtP_+XzXNaY-Cs{VWZtPMeySN-bE{x8gVmDm20?RUb^5+N8mfNjeff3Yacwh zXQ6mbaH++n%nWrV=kNEkRPHwDh3+ZZ&ZvK5z5801MT;Dic4RQaWUJJq`6GYj0}v6 z6%FJKWPu?m%f}+dBJwpUGf0BXN5K8Dm?LN0jjv5YT=S590E|m!pwq>d@2Lz>UN|HE z#Qeq4GaD~Wy>;u?&-|^f&VBg{Zr+jFP@Zw^^B>m9Z#HxuK5v&irBl(u!2G|)-Dy_I z#YSDrCnRLqMhjeu+~-g_wKd!K<}azMyxbeE?JfOmq*$uQm0&QzI4M4|_nSY{7K4lX aUkLAczwrJ9tC$`&f zqOZ(8Bi5~0mwmOh=efU@f6#-6#$o>ooBF>6J@?b#=HJVHa*I}I-h+CD=*T&S6SP$`!+ghorIcZ*(RTvLX zEAbYzbACS2RB3VHuaXCI)e;K7+<#vs86)s)5wD4|&c!puJEk)ow2b+&wfCO&%DI|p z=VwmSUfEdn^=1Zx@Bb+aUf7;XJJ)tVDDrdJmiCL>6V7FvHr~en^?mz+eKxN=l~pTk z3+~PgTebGmI>|;~=APU`>nkh(ieV*-k+hBJ?Vb* zpSN35+Ff2x5VybGHA$_wygfB$_BTQQP`!ncZ)Ix^)jG00ka>Qc{l)!%{f%owui89o zYm#B!x$)ZaRZRvrO?}0~=AG;L*mF&BwN2^nNMGf4_q);37iT@LJNSHhwr2)^MfZ|w zg|$jk#l3n;%Vkb~+vRQ5@TTEqfqBQm_-hdtFZ2a*2p?AnXVSm1=;5TsrwfeSU(AS; zE#9uM+FIOw@yuWKQ+LQ;wPRygpSQC8oz0OiC$2V{+-435sF#k)JRl2B)T|MDrUsV4 zgj@tn$aw}$jMo=1GchtTvDkkud11iI#;Mij(e|B}k&&B~!63_!+klgeIh2J>m?^{+ z#^DfVa&|NnF%SaDunThsC+3wTDmdpP79|=A8t{Wexr8}f5=#<;OA;$!y12ncNEnEL zd5d|tVK=aiS?p=RN4`&ZpvGIzQB@wu~>KhoXJcj>0M&v)J9VzwzsbD5YK85kEU z8ps>S0z*=kk420{!u<{|q47?;dIr;9J&QyHMVa7O%z z`HQ1xHeQ-~>(;NI`CDC``|=muyd$-tJmcEuKdh7AZ0J0E-Y$7cr=o>{`G1YO)2xz< zjk=akNXW8{7Pu6-&!Kc`YqsyrUs6|jxi?(fTl(2Zu~d&M!C-=MQha3ZH-Dxr1{e3g u5Z?2C;r$6#F+FO=s}C-`#Hg_2!GX3nhWw`v&yrZjliX3p^FL_4p(Oxq{5b#s literal 0 HcmV?d00001 diff --git a/tests/integration/long/test_ssl.py b/tests/integration/long/test_ssl.py new file mode 100644 index 0000000000..69e60db76f --- /dev/null +++ b/tests/integration/long/test_ssl.py @@ -0,0 +1,111 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import os +import ssl +from cassandra.cluster import Cluster +from cassandra import ConsistencyLevel +from cassandra.query import SimpleStatement +from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster + + +DEFAULT_PASSWORD = "cassandra" + +DEFAULT_SERVER_KEYSTORE_PATH = "tests/integration/long/ssl/keystore.jks" + +DEFAULT_CLIENT_CA_CERTS = 'tests/integration/long/ssl/driver_ca_cert.pem' + + +def setup_module(): + + """ + We need some custom setup for this module. This will start the ccm cluster with basic + ssl connectivity. No client authentication is performed at this time. + """ + + use_singledc(start=False) + ccm_cluster = get_cluster() + ccm_cluster.stop() + + # Fetch the absolute path to the keystore for ccm. + abs_path_server_keystore_path = os.path.abspath(DEFAULT_SERVER_KEYSTORE_PATH) + + # Configure ccm to use ssl. + config_options = {'client_encryption_options': {'enabled': True, + 'keystore': abs_path_server_keystore_path, + 'keystore_password': DEFAULT_PASSWORD}} + ccm_cluster.set_configuration_options(config_options) + ccm_cluster.start(wait_for_binary_proto=True, wait_other_notice=True) + + +def teardown_module(): + + """ + The rest of the tests don't need ssl enabled + reset the config options so as to not interfere with other tests. + """ + + ccm_cluster = get_cluster() + config_options = {} + ccm_cluster.set_configuration_options(config_options) + if ccm_cluster is not None: + ccm_cluster.stop() + + +class SSLConnectionTests(unittest.TestCase): + + def test_ssl_connection(self): + """ + Test to validate that we are able to connect to a cluster using ssl. + + test_ssl_connection Performs a simple sanity check to ensure that we can connect to a cluster with ssl. + + + @since 2.6.0 + @jira_ticket PYTHON-332 + @expected_result we can connect and preform some basic operations + + @test_category connection:ssl + """ + + # Setup temporary keyspace. + abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS) + + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, + 'ssl_version': ssl.PROTOCOL_TLSv1}) + self.session = self.cluster.connect() + + # attempt a few simple commands. + insert_keyspace = """CREATE KEYSPACE ssltest + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} + """ + statement = SimpleStatement(insert_keyspace) + statement.consistency_level = 3 + self.session.execute(statement) + + drop_keyspace = "DROP KEYSPACE ssltest" + + statement = SimpleStatement(drop_keyspace) + statement.consistency_level = ConsistencyLevel.ANY + self.session.execute(statement) + + + + From 0b6e25077b80747161ad759675b0a51944b20a47 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Wed, 17 Jun 2015 09:38:55 -0500 Subject: [PATCH 0214/2431] Updating protocol negotiation for better validation --- tests/integration/standard/test_cluster.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index cbd8cd0996..ee738e47a9 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -30,6 +30,7 @@ WhiteListRoundRobinPolicy) from cassandra.query import SimpleStatement, TraceUnavailable +from cassandra.protocol import MAX_SUPPORTED_VERSION from tests.integration import use_singledc, PROTOCOL_VERSION, get_server_versions, get_node, CASSANDRA_VERSION @@ -122,18 +123,23 @@ def test_protocol_negotiation(self): """ cluster = Cluster() + self.assertEqual(cluster.protocol_version, MAX_SUPPORTED_VERSION) session = cluster.connect() - default_protocol_version = session._protocol_version - + updated_protocol_version = session._protocol_version + updated_cluster_version = cluster.protocol_version # Make sure the correct protocol was selected by default if CASSANDRA_VERSION >= '2.2': - self.assertEqual(default_protocol_version, 4) + self.assertEqual(updated_protocol_version, 4) + self.assertEqual(updated_cluster_version, 4) elif CASSANDRA_VERSION >= '2.1': - self.assertEqual(default_protocol_version, 3) + self.assertEqual(updated_protocol_version, 3) + self.assertEqual(updated_cluster_version, 3) elif CASSANDRA_VERSION >= '2.0': - self.assertEqual(default_protocol_version, 2) + self.assertEqual(updated_protocol_version, 2) + self.assertEqual(updated_cluster_version, 2) else: - self.assertEqual(default_protocol_version, 1) + self.assertEqual(updated_protocol_version, 1) + self.assertEqual(updated_cluster_version, 1) cluster.shutdown() From 3315da86e68ad47638b253a8ddbf1da499ae9fe5 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 17 Jun 2015 09:46:52 -0500 Subject: [PATCH 0215/2431] Remove superfluous schema retry in CC connect Now that we're not waiting for agreement in the first place, this retry should not be needed. --- cassandra/cluster.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index b7d5061693..284314bee7 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2121,9 +2121,6 @@ def _try_connect(self, host): self._refresh_node_list_and_token_map(connection, preloaded_results=shared_results) self._refresh_schema(connection, preloaded_results=shared_results, schema_agreement_wait=-1) - if not self._cluster.metadata.keyspaces: - log.warning("[control connection] No schema built on connect; retrying without wait for schema agreement") - self._refresh_schema(connection, preloaded_results=shared_results, schema_agreement_wait=0) except Exception: connection.close() raise From 37cc21fd063ee9258db322ed0ec39edbc1b0633f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 17 Jun 2015 10:47:22 -0500 Subject: [PATCH 0216/2431] Make benchmark setup/teardown use minimal protocol version --- benchmarks/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/base.py b/benchmarks/base.py index 09e76a507d..3d02a150c2 100644 --- a/benchmarks/base.py +++ b/benchmarks/base.py @@ -61,7 +61,7 @@ def setup(hosts): log.info("Using 'cassandra' package from %s", cassandra.__path__) - cluster = Cluster(hosts) + cluster = Cluster(hosts, protocol_version=1) cluster.set_core_connections_per_host(HostDistance.LOCAL, 1) try: session = cluster.connect() @@ -89,7 +89,7 @@ def setup(hosts): def teardown(hosts): - cluster = Cluster(hosts) + cluster = Cluster(hosts, protocol_version=1) cluster.set_core_connections_per_host(HostDistance.LOCAL, 1) session = cluster.connect() session.execute("DROP KEYSPACE " + KEYSPACE) From 675e287a427c699c24b8ea24050519a7c7d858a6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 17 Jun 2015 13:02:32 -0500 Subject: [PATCH 0217/2431] Explicitly import gevent.sll in reactor impl Fixes an issue where ssl is not available at the package level, depending on what has executed thus far. --- cassandra/io/geventreactor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index a49fb737f6..5d8553cec6 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import gevent -from gevent import select, socket from gevent.event import Event from gevent.queue import Queue +from gevent import select, socket +import gevent.ssl from collections import defaultdict from functools import partial From df41a349f5a0883683be5ce1f543c5243caeb4b8 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Wed, 17 Jun 2015 18:51:15 -0700 Subject: [PATCH 0218/2431] retry cluster connection in SSL test --- tests/integration/long/test_ssl.py | 56 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/integration/long/test_ssl.py b/tests/integration/long/test_ssl.py index 69e60db76f..d7cfc5a841 100644 --- a/tests/integration/long/test_ssl.py +++ b/tests/integration/long/test_ssl.py @@ -12,29 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. - try: import unittest2 as unittest except ImportError: import unittest -import os -import ssl +import os, sys, traceback, logging, ssl from cassandra.cluster import Cluster from cassandra import ConsistencyLevel from cassandra.query import SimpleStatement -from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster +from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, remove_cluster +log = logging.getLogger(__name__) DEFAULT_PASSWORD = "cassandra" - DEFAULT_SERVER_KEYSTORE_PATH = "tests/integration/long/ssl/keystore.jks" - DEFAULT_CLIENT_CA_CERTS = 'tests/integration/long/ssl/driver_ca_cert.pem' def setup_module(): - """ We need some custom setup for this module. This will start the ccm cluster with basic ssl connectivity. No client authentication is performed at this time. @@ -56,31 +52,27 @@ def setup_module(): def teardown_module(): - """ - The rest of the tests don't need ssl enabled - reset the config options so as to not interfere with other tests. + The rest of the tests don't need ssl enabled, remove the cluster so as to not interfere with other tests. """ - ccm_cluster = get_cluster() - config_options = {} - ccm_cluster.set_configuration_options(config_options) - if ccm_cluster is not None: - ccm_cluster.stop() + remove_cluster() class SSLConnectionTests(unittest.TestCase): - def test_ssl_connection(self): + def test_can_connect_with_ssl_ca(self): """ - Test to validate that we are able to connect to a cluster using ssl. - - test_ssl_connection Performs a simple sanity check to ensure that we can connect to a cluster with ssl. + Test to validate that we are able to connect to a cluster using ssl. + test_can_connect_with_ssl_ca performs a simple sanity check to ensure that we can connect to a cluster with ssl + authentication via simple server-side shared certificate authority. The client is able to validate the identity + of the server, however by using this method the server can't trust the client unless additional authentication + has been provided. @since 2.6.0 @jira_ticket PYTHON-332 - @expected_result we can connect and preform some basic operations + @expected_result The client can connect via SSL and preform some basic operations @test_category connection:ssl """ @@ -88,9 +80,20 @@ def test_ssl_connection(self): # Setup temporary keyspace. abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS) - self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, + cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, 'ssl_version': ssl.PROTOCOL_TLSv1}) - self.session = self.cluster.connect() + tries = 0 + while True: + if tries > 5: + raise RuntimeError("Failed to connect to SSL cluster after 5 attempts") + try: + session = cluster.connect() + break + except Exception: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest @@ -98,14 +101,11 @@ def test_ssl_connection(self): """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 - self.session.execute(statement) + session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" - statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY - self.session.execute(statement) - - - + session.execute(statement) + cluster.shutdown() \ No newline at end of file From 2cb42b307b3dd5b3b4bec7239cd8b7bc3039db47 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 18 Jun 2015 11:29:20 -0500 Subject: [PATCH 0219/2431] benchmarks: Raise log level for cassandra package Reduce node discovery noise in default output. --- benchmarks/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/base.py b/benchmarks/base.py index 3d02a150c2..65df52864f 100644 --- a/benchmarks/base.py +++ b/benchmarks/base.py @@ -36,6 +36,8 @@ handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")) log.addHandler(handler) +logging.getLogger('cassandra').setLevel(logging.WARN) + have_libev = False supported_reactors = [AsyncoreConnection] try: From d7929bcd2e0a865c758d001578d36cb3cbff5fbf Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 18 Jun 2015 16:50:54 -0700 Subject: [PATCH 0220/2431] more fixes for jenkins test stabilization --- tests/integration/long/test_failure_types.py | 17 +++++++++++++++-- tests/integration/long/test_ssl.py | 4 ++-- tests/integration/standard/test_cluster.py | 2 +- tests/integration/standard/test_metadata.py | 6 +++--- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/integration/long/test_failure_types.py b/tests/integration/long/test_failure_types.py index 64d06f46a3..30e05b60b6 100644 --- a/tests/integration/long/test_failure_types.py +++ b/tests/integration/long/test_failure_types.py @@ -81,6 +81,19 @@ def tearDown(self): # Restart the nodes to fully functional again self.setFailingNodes(failing_nodes, "testksfail") + def execute_helper(self, session, query): + tries = 0 + while tries < 100: + try: + return session.execute(query) + except OperationTimedOut: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 + + raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(query)) + def execute_concurrent_args_helper(self, session, query, params): tries = 0 while tries < 100: @@ -129,10 +142,10 @@ def _perform_cql_statement(self, text, consistency_level, expected_exception): statement.consistency_level = consistency_level if expected_exception is None: - self.session.execute(statement) + self.execute_helper(self.session, statement) else: with self.assertRaises(expected_exception): - self.session.execute(statement) + self.execute_helper(self.session, statement) def test_write_failures_from_coordinator(self): """ diff --git a/tests/integration/long/test_ssl.py b/tests/integration/long/test_ssl.py index d7cfc5a841..931e6d112c 100644 --- a/tests/integration/long/test_ssl.py +++ b/tests/integration/long/test_ssl.py @@ -80,13 +80,13 @@ def test_can_connect_with_ssl_ca(self): # Setup temporary keyspace. abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS) - cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, - 'ssl_version': ssl.PROTOCOL_TLSv1}) tries = 0 while True: if tries > 5: raise RuntimeError("Failed to connect to SSL cluster after 5 attempts") try: + cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, + 'ssl_version': ssl.PROTOCOL_TLSv1}) session = cluster.connect() break except Exception: diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index df41b67ed1..b8439c4de3 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -352,7 +352,7 @@ def test_refresh_schema_type(self): current_test1rf_meta = current_meta[keyspace_name] current_type_meta = current_test1rf_meta.user_types[type_name] self.assertIs(original_meta, current_meta) - self.assertIs(original_test1rf_meta, current_test1rf_meta) + self.assertEqual(original_test1rf_meta.export_as_string(), current_test1rf_meta.export_as_string()) self.assertIsNot(original_type_meta, current_type_meta) self.assertEqual(original_type_meta.as_cql_query(), current_type_meta.as_cql_query()) session.shutdown() diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index d9b67e9963..de89e8c507 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -255,12 +255,12 @@ def test_composite_in_compound_primary_key_ordering(self): def test_indexes(self): create_statement = self.make_create_statement(["a"], ["b", "c"], ["d", "e", "f"]) create_statement += " WITH CLUSTERING ORDER BY (b ASC, c ASC)" - self.session.execute(create_statement) + execute_until_pass(self.session, create_statement) d_index = "CREATE INDEX d_index ON %s.%s (d)" % (self.ksname, self.cfname) e_index = "CREATE INDEX e_index ON %s.%s (e)" % (self.ksname, self.cfname) - self.session.execute(d_index) - self.session.execute(e_index) + execute_until_pass(self.session, d_index) + execute_until_pass(self.session, e_index) tablemeta = self.get_table_metadata() statements = tablemeta.export_as_string().strip() From 93331e0e2243a3d21d3cd515757ce8ebadb7047d Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Thu, 18 Jun 2015 18:58:44 -0700 Subject: [PATCH 0221/2431] a few more test stabilizations --- tests/integration/standard/test_types.py | 8 ++++---- tests/integration/standard/test_udts.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index 3dd87e82b9..f5c148ddd4 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -261,7 +261,7 @@ def test_can_insert_empty_strings_and_nulls(self): if datatype in string_types: string_columns.add(col_name) - s.execute("CREATE TABLE all_empty ({0})".format(', '.join(alpha_type_list))) + execute_until_pass(s, "CREATE TABLE all_empty ({0})".format(', '.join(alpha_type_list))) # verify all types initially null with simple statement columns_string = ','.join(col_names) @@ -349,11 +349,11 @@ def test_can_insert_empty_values_for_int32(self): """ s = self.session - s.execute("CREATE TABLE empty_values (a text PRIMARY KEY, b int)") - s.execute("INSERT INTO empty_values (a, b) VALUES ('a', blobAsInt(0x))") + execute_until_pass(s, "CREATE TABLE empty_values (a text PRIMARY KEY, b int)") + execute_until_pass(s, "INSERT INTO empty_values (a, b) VALUES ('a', blobAsInt(0x))") try: Int32Type.support_empty_values = True - results = s.execute("SELECT b FROM empty_values WHERE a='a'")[0] + results = execute_until_pass(s, "SELECT b FROM empty_values WHERE a='a'")[0] self.assertIs(EMPTY, results.b) finally: Int32Type.support_empty_values = False diff --git a/tests/integration/standard/test_udts.py b/tests/integration/standard/test_udts.py index 7d9eb99173..eeef0508ba 100644 --- a/tests/integration/standard/test_udts.py +++ b/tests/integration/standard/test_udts.py @@ -326,21 +326,21 @@ def test_can_insert_udts_with_varying_lengths(self): def nested_udt_schema_helper(self, session, MAX_NESTING_DEPTH): # create the seed udt - session.execute("CREATE TYPE depth_0 (age int, name text)") + execute_until_pass(session, "CREATE TYPE depth_0 (age int, name text)") # create the nested udts for i in range(MAX_NESTING_DEPTH): - session.execute("CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) + execute_until_pass(session, "CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) # create a table with multiple sizes of nested udts # no need for all nested types, only a spot checked few and the largest one - session.execute("CREATE TABLE mytable (" - "k int PRIMARY KEY, " - "v_0 frozen, " - "v_1 frozen, " - "v_2 frozen, " - "v_3 frozen, " - "v_{0} frozen)".format(MAX_NESTING_DEPTH)) + execute_until_pass(session, "CREATE TABLE mytable (" + "k int PRIMARY KEY, " + "v_0 frozen, " + "v_1 frozen, " + "v_2 frozen, " + "v_3 frozen, " + "v_{0} frozen)".format(MAX_NESTING_DEPTH)) def nested_udt_creation_helper(self, udts, i): if i == 0: From ee2243c217e6ada6275845f3cb8f5c3f8a7e8e7b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 19 Jun 2015 10:08:48 -0500 Subject: [PATCH 0222/2431] Revert "Revert "Merge pull request #298 from datastax/PYTHON-108"" This reverts commit dfa91b8bd5d6dcfe13ce3f4caf4eb721a7dd3d18. Conflicts: cassandra/cluster.py cassandra/connection.py cassandra/io/asyncorereactor.py cassandra/io/eventletreactor.py cassandra/io/geventreactor.py cassandra/io/libevreactor.py cassandra/io/twistedreactor.py cassandra/query.py --- cassandra/cluster.py | 92 +++++++++++--------- cassandra/connection.py | 105 +++++++++++++++++++++-- cassandra/io/asyncorereactor.py | 62 +++++--------- cassandra/io/eventletreactor.py | 55 +++++++----- cassandra/io/geventreactor.py | 55 +++++++----- cassandra/io/libevreactor.py | 60 ++++++++----- cassandra/io/libevwrapper.c | 133 +++++++++++++++++++++++++++++ cassandra/io/twistedreactor.py | 61 +++++++------ cassandra/query.py | 6 +- docs/object_mapper.rst | 2 +- tests/unit/test_connection.py | 3 - tests/unit/test_response_future.py | 22 ++--- 12 files changed, 460 insertions(+), 196 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 828fa8266b..45784c4623 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1442,8 +1442,7 @@ class Session(object): """ A default timeout, measured in seconds, for queries executed through :meth:`.execute()` or :meth:`.execute_async()`. This default may be - overridden with the `timeout` parameter for either of those methods - or the `timeout` parameter for :meth:`.ResponseFuture.result()`. + overridden with the `timeout` parameter for either of those methods. Setting this to :const:`None` will cause no timeouts to be set by default. @@ -1581,17 +1580,14 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False, custom_ If `query` is a Statement with its own custom_payload. The message payload will be a union of the two, with the values specified here taking precedence. """ - if timeout is _NOT_SET: - timeout = self.default_timeout - if trace and not isinstance(query, Statement): raise TypeError( "The query argument must be an instance of a subclass of " "cassandra.query.Statement when trace=True") - future = self.execute_async(query, parameters, trace, custom_payload) + future = self.execute_async(query, parameters, trace, custom_payload, timeout) try: - result = future.result(timeout) + result = future.result() finally: if trace: try: @@ -1601,7 +1597,7 @@ def execute(self, query, parameters=None, timeout=_NOT_SET, trace=False, custom_ return result - def execute_async(self, query, parameters=None, trace=False, custom_payload=None): + def execute_async(self, query, parameters=None, trace=False, custom_payload=None, timeout=_NOT_SET): """ Execute the given query and return a :class:`~.ResponseFuture` object which callbacks may be attached to for asynchronous response @@ -1646,11 +1642,14 @@ def execute_async(self, query, parameters=None, trace=False, custom_payload=None ... log.exception("Operation failed:") """ - future = self._create_response_future(query, parameters, trace, custom_payload) + if timeout is _NOT_SET: + timeout = self.default_timeout + + future = self._create_response_future(query, parameters, trace, custom_payload, timeout) future.send_request() return future - def _create_response_future(self, query, parameters, trace, custom_payload): + def _create_response_future(self, query, parameters, trace, custom_payload, timeout): """ Returns the ResponseFuture before calling send_request() on it """ prepared_statement = None @@ -1704,7 +1703,7 @@ def _create_response_future(self, query, parameters, trace, custom_payload): message.update_custom_payload(custom_payload) return ResponseFuture( - self, message, query, self.default_timeout, metrics=self._metrics, + self, message, query, timeout, metrics=self._metrics, prepared_statement=prepared_statement) def prepare(self, query, custom_payload=None): @@ -1737,11 +1736,10 @@ def prepare(self, query, custom_payload=None): message. See :ref:`custom_payload`. """ message = PrepareMessage(query=query) - message.custom_payload = custom_payload - future = ResponseFuture(self, message, query=None) + future = ResponseFuture(self, message, query=None, timeout=self.default_timeout) try: future.send_request() - query_id, column_metadata, pk_indexes = future.result(self.default_timeout) + query_id, column_metadata, pk_indexes = future.result() except Exception: log.exception("Error preparing query:") raise @@ -1767,7 +1765,7 @@ def prepare_on_all_hosts(self, query, excluded_host): futures = [] for host in self._pools.keys(): if host != excluded_host and host.is_up: - future = ResponseFuture(self, PrepareMessage(query=query), None) + future = ResponseFuture(self, PrepareMessage(query=query), None, self.default_timeout) # we don't care about errors preparing against specific hosts, # since we can always prepare them as needed when the prepared @@ -1788,7 +1786,7 @@ def prepare_on_all_hosts(self, query, excluded_host): for host, future in futures: try: - future.result(self.default_timeout) + future.result() except Exception: log.exception("Error preparing query for host %s:", host) @@ -2832,13 +2830,14 @@ class ResponseFuture(object): _paging_state = None _custom_payload = None _warnings = None + _timer = None - def __init__(self, session, message, query, default_timeout=None, metrics=None, prepared_statement=None): + def __init__(self, session, message, query, timeout, metrics=None, prepared_statement=None): self.session = session self.row_factory = session.row_factory self.message = message self.query = query - self.default_timeout = default_timeout + self.timeout = timeout self._metrics = metrics self.prepared_statement = prepared_statement self._callback_lock = Lock() @@ -2849,6 +2848,18 @@ def __init__(self, session, message, query, default_timeout=None, metrics=None, self._errors = {} self._callbacks = [] self._errbacks = [] + self._start_timer() + + def _start_timer(self): + if self.timeout is not None: + self._timer = self.session.cluster.connection_class.create_timer(self.timeout, self._on_timeout) + + def _cancel_timer(self): + if self._timer: + self._timer.cancel() + + def _on_timeout(self): + self._set_final_exception(OperationTimedOut(self._errors, self._current_host)) def _make_query_plan(self): # convert the list/generator/etc to an iterator so that subsequent @@ -2973,6 +2984,7 @@ def start_fetching_next_page(self): self._event.clear() self._final_result = _NOT_SET self._final_exception = None + self._start_timer() self.send_request() def _reprepare(self, prepare_message): @@ -3187,6 +3199,7 @@ def _execute_after_prepare(self, response): "statement on host %s: %s" % (self._current_host, response))) def _set_final_result(self, response): + self._cancel_timer() if self._metrics is not None: self._metrics.request_timer.addValue(time.time() - self._start_time) @@ -3201,6 +3214,7 @@ def _set_final_result(self, response): fn(response, *args, **kwargs) def _set_final_exception(self, response): + self._cancel_timer() if self._metrics is not None: self._metrics.request_timer.addValue(time.time() - self._start_time) @@ -3244,6 +3258,11 @@ def result(self, timeout=_NOT_SET): encountered. If the final result or error has not been set yet, this method will block until that time. + .. versionchanged:: 2.6.0 + + **`timeout` is deprecated. Use timeout in the Session execute functions instead. + The following description applies to deprecated behavior:** + You may set a timeout (in seconds) with the `timeout` parameter. By default, the :attr:`~.default_timeout` for the :class:`.Session` this was created through will be used for the timeout on this @@ -3257,11 +3276,6 @@ def result(self, timeout=_NOT_SET): This is a client-side timeout. For more information about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`. - **Important**: This timeout currently has no effect on callbacks registered - on a :class:`~.ResponseFuture` through :meth:`.ResponseFuture.add_callback` or - :meth:`.ResponseFuture.add_errback`; even if a query exceeds this default - timeout, neither the registered callback or errback will be called. - Example usage:: >>> future = session.execute_async("SELECT * FROM mycf") @@ -3275,27 +3289,24 @@ def result(self, timeout=_NOT_SET): ... log.exception("Operation failed:") """ - if timeout is _NOT_SET: - timeout = self.default_timeout + if timeout is not _NOT_SET: + msg = "ResponseFuture.result timeout argument is deprecated. Specify the request timeout via Session.execute[_async]." + warnings.warn(msg, DeprecationWarning) + log.warning(msg) + else: + timeout = None + self._event.wait(timeout) + # TODO: remove this conditional when deprecated timeout parameter is removed + if not self._event.is_set(): + self._on_timeout() if self._final_result is not _NOT_SET: if self._paging_state is None: return self._final_result else: - return PagedResult(self, self._final_result, timeout) - elif self._final_exception: - raise self._final_exception + return PagedResult(self, self._final_result) else: - self._event.wait(timeout=timeout) - if self._final_result is not _NOT_SET: - if self._paging_state is None: - return self._final_result - else: - return PagedResult(self, self._final_result, timeout) - elif self._final_exception: - raise self._final_exception - else: - raise OperationTimedOut(errors=self._errors, last_host=self._current_host) + raise self._final_exception def get_query_trace(self, max_wait=None): """ @@ -3450,10 +3461,9 @@ class will be returned. response_future = None - def __init__(self, response_future, initial_response, timeout=_NOT_SET): + def __init__(self, response_future, initial_response): self.response_future = response_future self.current_response = iter(initial_response) - self.timeout = timeout def __iter__(self): return self @@ -3466,7 +3476,7 @@ def next(self): raise self.response_future.start_fetching_next_page() - result = self.response_future.result(self.timeout) + result = self.response_future.result() if self.response_future.has_more_pages: self.current_response = result.current_response else: diff --git a/cassandra/connection.py b/cassandra/connection.py index 464ec792ba..95858bd8c1 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -16,6 +16,7 @@ from collections import defaultdict, deque, namedtuple import errno from functools import wraps, partial +from heapq import heappush, heappop import io import logging import socket @@ -44,7 +45,8 @@ QueryMessage, ResultMessage, decode_response, InvalidRequestException, SupportedMessage, AuthResponseMessage, AuthChallengeMessage, - AuthSuccessMessage, ProtocolException, MAX_SUPPORTED_VERSION) + AuthSuccessMessage, ProtocolException, + MAX_SUPPORTED_VERSION, RegisterMessage) from cassandra.util import OrderedDict @@ -219,6 +221,7 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.is_control_connection = is_control_connection self.user_type_map = user_type_map self._push_watchers = defaultdict(set) + self._callbacks = {} self._iobuf = io.BytesIO() if protocol_version >= 3: @@ -233,6 +236,7 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.highest_request_id = self.max_request_id self.lock = RLock() + self.connected_event = Event() @classmethod def initialize_reactor(self): @@ -250,6 +254,10 @@ def handle_fork(self): """ pass + @classmethod + def create_timer(cls, timeout, callback): + raise NotImplementedError() + @classmethod def factory(cls, host, timeout, *args, **kwargs): """ @@ -407,11 +415,24 @@ def wait_for_responses(self, *msgs, **kwargs): self.defunct(exc) raise - def register_watcher(self, event_type, callback): - raise NotImplementedError() + def register_watcher(self, event_type, callback, register_timeout=None): + """ + Register a callback for a given event type. + """ + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=[event_type]), + timeout=register_timeout) - def register_watchers(self, type_callback_dict): - raise NotImplementedError() + def register_watchers(self, type_callback_dict, register_timeout=None): + """ + Register multiple callback/event type pairs, expressed as a dict. + """ + for event_type, callback in type_callback_dict.items(): + self._push_watchers[event_type].add(callback) + self.wait_for_response( + RegisterMessage(event_list=type_callback_dict.keys()), + timeout=register_timeout) def control_conn_disposed(self): self.is_control_connection = False @@ -907,3 +928,77 @@ def stop(self): def _raise_if_stopped(self): if self._shutdown_event.is_set(): raise self.ShutdownException() + + +class Timer(object): + + canceled = False + + def __init__(self, timeout, callback): + self.end = time.time() + timeout + self.callback = callback + if timeout < 0: + self.callback() + + def cancel(self): + self.canceled = True + + def finish(self, time_now): + if self.canceled: + return True + + if time_now >= self.end: + self.callback() + return True + + return False + + +class TimerManager(object): + + def __init__(self): + self._queue = [] + self._new_timers = [] + + def add_timer(self, timer): + """ + called from client thread with a Timer object + """ + self._new_timers.append((timer.end, timer)) + + def service_timeouts(self): + """ + run callbacks on all expired timers + Called from the event thread + :return: next end time, or None + """ + queue = self._queue + new_timers = self._new_timers + while new_timers: + heappush(queue, new_timers.pop()) + + now = time.time() + while queue: + try: + timer = queue[0][1] + if timer.finish(now): + heappop(queue) + else: + return timer.end + except Exception: + log.exception("Exception while servicing timeout callback: ") + + @property + def next_timeout(self): + try: + return self._queue[0][0] + except IndexError: + pass + + @property + def next_offset(self): + try: + next_end = self._queue[0][0] + return next_end - time.time() + except IndexError: + pass diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 616b5c0000..fae87a7354 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -19,6 +19,7 @@ import socket import sys from threading import Event, Lock, Thread +import time import weakref from six.moves import range @@ -35,8 +36,9 @@ except ImportError: ssl = None # NOQA -from cassandra.connection import (Connection, ConnectionShutdown, NONBLOCKING) -from cassandra.protocol import RegisterMessage +from cassandra.connection import (Connection, ConnectionShutdown, + ConnectionException, NONBLOCKING, + Timer, TimerManager) log = logging.getLogger(__name__) @@ -52,15 +54,17 @@ def _cleanup(loop_weakref): class AsyncoreLoop(object): + def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() self._started = False self._shutdown = False - self._conns_lock = Lock() - self._conns = WeakSet() self._thread = None + + self._timers = TimerManager() + atexit.register(partial(_cleanup, weakref.ref(self))) def maybe_start(self): @@ -83,24 +87,22 @@ def maybe_start(self): def _run_loop(self): log.debug("Starting asyncore event loop") with self._loop_lock: - while True: + while not self._shutdown: try: - asyncore.loop(timeout=0.001, use_poll=True, count=1000) + asyncore.loop(timeout=0.001, use_poll=True, count=100) + self._timers.service_timeouts() + if not asyncore.socket_map: + time.sleep(0.005) except Exception: log.debug("Asyncore event loop stopped unexepectedly", exc_info=True) break - - if self._shutdown: - break - - with self._conns_lock: - if len(self._conns) == 0: - break - self._started = False log.debug("Asyncore event loop ended") + def add_timer(self, timer): + self._timers.add_timer(timer) + def _cleanup(self): self._shutdown = True if not self._thread: @@ -115,14 +117,6 @@ def _cleanup(self): log.debug("Event loop thread was joined") - def connection_created(self, connection): - with self._conns_lock: - self._conns.add(connection) - - def connection_destroyed(self, connection): - with self._conns_lock: - self._conns.discard(connection) - class AsyncoreConnection(Connection, asyncore.dispatcher): """ @@ -152,18 +146,19 @@ def handle_fork(cls): cls._loop._cleanup() cls._loop = None + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._loop.add_timer(timer) + return timer + def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) asyncore.dispatcher.__init__(self) - self.connected_event = Event() - - self._callbacks = {} self.deque = deque() self.deque_lock = Lock() - self._loop.connection_created(self) - self._connect_socket() asyncore.dispatcher.__init__(self, self._socket) @@ -187,8 +182,6 @@ def close(self): asyncore.dispatcher.close(self) log.debug("Closed socket to %s", self.host) - self._loop.connection_destroyed(self) - if not self.is_defunct: self.error_all_callbacks( ConnectionShutdown("Connection to %s was closed" % self.host)) @@ -267,14 +260,3 @@ def writable(self): def readable(self): return self._readable or (self.is_control_connection and not (self.is_defunct or self.is_closed)) - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), timeout=register_timeout) diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index e65d5aa544..b0206f43d2 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -16,7 +16,6 @@ # Originally derived from MagnetoDB source: # https://github.com/stackforge/magnetodb/blob/2015.1.0b1/magnetodb/common/cassandra/io/eventletreactor.py -from collections import defaultdict from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL import eventlet from eventlet.green import select, socket @@ -24,11 +23,12 @@ from functools import partial import logging import os -from six.moves import xrange from threading import Event +import time + +from six.moves import xrange -from cassandra.connection import Connection, ConnectionShutdown -from cassandra.protocol import RegisterMessage +from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager log = logging.getLogger(__name__) @@ -52,19 +52,45 @@ class EventletConnection(Connection): _socket_impl = eventlet.green.socket _ssl_impl = eventlet.green.ssl + _timers = None + _timeout_watcher = None + _new_timer = None + @classmethod def initialize_reactor(cls): eventlet.monkey_patch() + if not cls._timers: + cls._timers = TimerManager() + cls._timeout_watcher = eventlet.spawn(cls.service_timeouts) + cls._new_timer = Event() + + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._timers.add_timer(timer) + cls._new_timer.set() + return timer + + @classmethod + def service_timeouts(cls): + """ + cls._timeout_watcher runs in this loop forever. + It is usually waiting for the next timeout on the cls._new_timer Event. + When new timers are added, that event is set so that the watcher can + wake up and possibly set an earlier timeout. + """ + timer_manager = cls._timers + while True: + next_end = timer_manager.service_timeouts() + sleep_time = max(next_end - time.time(), 0) if next_end else 10000 + cls._new_timer.wait(sleep_time) + cls._new_timer.clear() def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() self._write_queue = Queue() - self._callbacks = {} - self._push_watchers = defaultdict(set) - self._connect_socket() self._read_watcher = eventlet.spawn(lambda: self.handle_read()) @@ -142,16 +168,3 @@ def push(self, data): chunk_size = self.out_buffer_size for i in xrange(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 5d8553cec6..60a3f2d031 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import gevent -from gevent.event import Event +import gevent.event from gevent.queue import Queue from gevent import select, socket import gevent.ssl @@ -21,13 +21,13 @@ from functools import partial import logging import os +import time -from six.moves import xrange +from six.moves import range from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL -from cassandra.connection import Connection, ConnectionShutdown -from cassandra.protocol import RegisterMessage +from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager log = logging.getLogger(__name__) @@ -51,15 +51,39 @@ class GeventConnection(Connection): _socket_impl = gevent.socket _ssl_impl = gevent.ssl + _timers = None + _timeout_watcher = None + _new_timer = None + + @classmethod + def initialize_reactor(cls): + if not cls._timers: + cls._timers = TimerManager() + cls._timeout_watcher = gevent.spawn(cls.service_timeouts) + cls._new_timer = gevent.event.Event() + + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._timers.add_timer(timer) + cls._new_timer.set() + return timer + + @classmethod + def service_timeouts(cls): + timer_manager = cls._timers + timer_event = cls._new_timer + while True: + next_end = timer_manager.service_timeouts() + sleep_time = max(next_end - time.time(), 0) if next_end else 10000 + timer_event.wait(sleep_time) + timer_event.clear() + def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() self._write_queue = Queue() - self._callbacks = {} - self._push_watchers = defaultdict(set) - self._connect_socket() self._read_watcher = gevent.spawn(self.handle_read) @@ -142,18 +166,5 @@ def handle_read(self): def push(self, data): chunk_size = self.out_buffer_size - for i in xrange(0, len(data), chunk_size): + for i in range(0, len(data), chunk_size): self._write_queue.put(data[i:i + chunk_size]) - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 00127a1fa2..af0c3a6015 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -17,13 +17,14 @@ import logging import os import socket -from threading import Event, Lock, Thread +import ssl +from threading import Lock, Thread import weakref -from six.moves import xrange +from six.moves import range -from cassandra.connection import Connection, ConnectionShutdown, NONBLOCKING, ssl -from cassandra.protocol import RegisterMessage +from cassandra.connection import (Connection, ConnectionShutdown, + NONBLOCKING, Timer, TimerManager) try: import cassandra.io.libevwrapper as libev except ImportError: @@ -44,7 +45,6 @@ def _cleanup(loop_weakref): loop = loop_weakref() except ReferenceError: return - loop._cleanup() @@ -79,10 +79,10 @@ def __init__(self): self._loop.unref() self._preparer.start() - atexit.register(partial(_cleanup, weakref.ref(self))) + self._timers = TimerManager() + self._loop_timer = libev.Timer(self._loop, self._on_loop_timer) - def notify(self): - self._notifier.send() + atexit.register(partial(_cleanup, weakref.ref(self))) def maybe_start(self): should_start = False @@ -127,6 +127,7 @@ def _cleanup(self): conn._read_watcher.stop() del conn._read_watcher + self.notify() # wake the timer watcher log.debug("Waiting for event loop thread to join...") self._thread.join(timeout=1.0) if self._thread.is_alive(): @@ -137,6 +138,24 @@ def _cleanup(self): log.debug("Event loop thread was joined") self._loop = None + def add_timer(self, timer): + self._timers.add_timer(timer) + self._notifier.send() # wake up in case this timer is earlier + + def _update_timer(self): + if not self._shutdown: + self._timers.service_timeouts() + offset = self._timers.next_offset or 100000 # none pending; will be updated again when something new happens + self._loop_timer.start(offset) + else: + self._loop_timer.stop() + + def _on_loop_timer(self): + self._timers.service_timeouts() + + def notify(self): + self._notifier.send() + def connection_created(self, conn): with self._conn_set_lock: new_live_conns = self._live_conns.copy() @@ -199,6 +218,9 @@ def _loop_will_run(self, prepare): changed = True + # TODO: update to do connection management, timer updates through dedicated async 'notifier' callbacks + self._update_timer() + if changed: self._notifier.send() @@ -229,12 +251,15 @@ def handle_fork(cls): cls._libevloop._cleanup() cls._libevloop = None + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._libevloop.add_timer(timer) + return timer + def __init__(self, *args, **kwargs): Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() - - self._callbacks = {} self.deque = deque() self._deque_lock = Lock() self._connect_socket() @@ -332,7 +357,7 @@ def push(self, data): sabs = self.out_buffer_size if len(data) > sabs: chunks = [] - for i in xrange(0, len(data), sabs): + for i in range(0, len(data), sabs): chunks.append(data[i:i + sabs]) else: chunks = [data] @@ -340,14 +365,3 @@ def push(self, data): with self._deque_lock: self.deque.extend(chunks) self._libevloop.notify() - - def register_watcher(self, event_type, callback, register_timeout=None): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), timeout=register_timeout) diff --git a/cassandra/io/libevwrapper.c b/cassandra/io/libevwrapper.c index cbac83b277..99e1df30f7 100644 --- a/cassandra/io/libevwrapper.c +++ b/cassandra/io/libevwrapper.c @@ -451,6 +451,131 @@ static PyTypeObject libevwrapper_PrepareType = { (initproc)Prepare_init, /* tp_init */ }; +typedef struct libevwrapper_Timer { + PyObject_HEAD + struct ev_timer timer; + struct libevwrapper_Loop *loop; + PyObject *callback; +} libevwrapper_Timer; + +static void +Timer_dealloc(libevwrapper_Timer *self) { + Py_XDECREF(self->loop); + Py_XDECREF(self->callback); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static void timer_callback(struct ev_loop *loop, ev_timer *watcher, int revents) { + libevwrapper_Timer *self = watcher->data; + + PyObject *result = NULL; + PyGILState_STATE gstate; + + gstate = PyGILState_Ensure(); + result = PyObject_CallFunction(self->callback, NULL); + if (!result) { + PyErr_WriteUnraisable(self->callback); + } + Py_XDECREF(result); + + PyGILState_Release(gstate); +} + +static int +Timer_init(libevwrapper_Timer *self, PyObject *args, PyObject *kwds) { + PyObject *callback; + PyObject *loop; + + if (!PyArg_ParseTuple(args, "OO", &loop, &callback)) { + return -1; + } + + if (loop) { + Py_INCREF(loop); + self->loop = (libevwrapper_Loop *)loop; + } else { + return -1; + } + + if (callback) { + if (!PyCallable_Check(callback)) { + PyErr_SetString(PyExc_TypeError, "callback parameter must be callable"); + Py_XDECREF(loop); + return -1; + } + Py_INCREF(callback); + self->callback = callback; + } + ev_init(&self->timer, timer_callback); + self->timer.data = self; + return 0; +} + +static PyObject * +Timer_start(libevwrapper_Timer *self, PyObject *args) { + double timeout; + if (!PyArg_ParseTuple(args, "d", &timeout)) { + return NULL; + } + /* some tiny non-zero number to avoid zero, and + make it run immediately for negative timeouts */ + self->timer.repeat = fmax(timeout, 0.000000001); + ev_timer_again(self->loop->loop, &self->timer); + Py_RETURN_NONE; +} + +static PyObject * +Timer_stop(libevwrapper_Timer *self, PyObject *args) { + ev_timer_stop(self->loop->loop, &self->timer); + Py_RETURN_NONE; +} + +static PyMethodDef Timer_methods[] = { + {"start", (PyCFunction)Timer_start, METH_VARARGS, "Start the Timer watcher"}, + {"stop", (PyCFunction)Timer_stop, METH_NOARGS, "Stop the Timer watcher"}, + {NULL} /* Sentinal */ +}; + +static PyTypeObject libevwrapper_TimerType = { + PyVarObject_HEAD_INIT(NULL, 0) + "cassandra.io.libevwrapper.Timer", /*tp_name*/ + sizeof(libevwrapper_Timer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)Timer_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "Timer objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Timer_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)Timer_init, /* tp_init */ +}; + + static PyMethodDef module_methods[] = { {NULL} /* Sentinal */ }; @@ -500,6 +625,10 @@ initlibevwrapper(void) if (PyType_Ready(&libevwrapper_AsyncType) < 0) INITERROR; + libevwrapper_TimerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&libevwrapper_TimerType) < 0) + INITERROR; + # if PY_MAJOR_VERSION >= 3 module = PyModule_Create(&moduledef); # else @@ -532,6 +661,10 @@ initlibevwrapper(void) if (PyModule_AddObject(module, "Async", (PyObject *)&libevwrapper_AsyncType) == -1) INITERROR; + Py_INCREF(&libevwrapper_TimerType); + if (PyModule_AddObject(module, "Timer", (PyObject *)&libevwrapper_TimerType) == -1) + INITERROR; + if (!PyEval_ThreadsInitialized()) { PyEval_InitThreads(); } diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index 1de8de1554..b02fb6ab32 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -15,15 +15,15 @@ Module that implements an event loop based on twisted ( https://twistedmatrix.com ). """ -from twisted.internet import reactor, protocol -from threading import Event, Thread, Lock +import atexit from functools import partial import logging +from threading import Thread, Lock +import time +from twisted.internet import reactor, protocol import weakref -import atexit -from cassandra.connection import Connection, ConnectionShutdown -from cassandra.protocol import RegisterMessage +from cassandra.connection import Connection, ConnectionShutdown, Timer, TimerManager log = logging.getLogger(__name__) @@ -108,9 +108,12 @@ class TwistedLoop(object): _lock = None _thread = None + _timeout_task = None + _timeout = None def __init__(self): self._lock = Lock() + self._timers = TimerManager() def maybe_start(self): with self._lock: @@ -132,6 +135,27 @@ def _cleanup(self): "Cluster.shutdown() to avoid this.") log.debug("Event loop thread was joined") + def add_timer(self, timer): + self._timers.add_timer(timer) + # callFromThread to schedule from the loop thread, where + # the timeout task can safely be modified + reactor.callFromThread(self._schedule_timeout, timer.end) + + def _schedule_timeout(self, next_timeout): + if next_timeout: + delay = max(next_timeout - time.time(), 0) + if self._timeout_task and self._timeout_task.active(): + if next_timeout < self._timeout: + self._timeout_task.reset(delay) + self._timeout = next_timeout + else: + self._timeout_task = reactor.callLater(delay, self._on_loop_timer) + self._timeout = next_timeout + + def _on_loop_timer(self): + self._timers.service_timeouts() + self._schedule_timeout(self._timers.next_timeout) + class TwistedConnection(Connection): """ @@ -146,6 +170,12 @@ def initialize_reactor(cls): if not cls._loop: cls._loop = TwistedLoop() + @classmethod + def create_timer(cls, timeout, callback): + timer = Timer(timeout, callback) + cls._loop.add_timer(timer) + return timer + def __init__(self, *args, **kwargs): """ Initialization method. @@ -157,11 +187,9 @@ def __init__(self, *args, **kwargs): """ Connection.__init__(self, *args, **kwargs) - self.connected_event = Event() self.is_closed = True self.connector = None - self._callbacks = {} reactor.callFromThread(self.add_connection) self._loop.maybe_start() @@ -218,22 +246,3 @@ def push(self, data): the event loop when it gets the chance. """ reactor.callFromThread(self.connector.transport.write, data) - - def register_watcher(self, event_type, callback, register_timeout=None): - """ - Register a callback for a given event type. - """ - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=[event_type]), - timeout=register_timeout) - - def register_watchers(self, type_callback_dict, register_timeout=None): - """ - Register multiple callback/event type pairs, expressed as a dict. - """ - for event_type, callback in type_callback_dict.items(): - self._push_watchers[event_type].add(callback) - self.wait_for_response( - RegisterMessage(event_list=type_callback_dict.keys()), - timeout=register_timeout) diff --git a/cassandra/query.py b/cassandra/query.py index 6709cbeff6..21b668dfb3 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -917,14 +917,14 @@ def populate(self, max_wait=2.0): break def _execute(self, query, parameters, time_spent, max_wait): + timeout = (max_wait - time_spent) if max_wait is not None else None + future = self._session._create_response_future(query, parameters, trace=False, custom_payload=None, timeout=timeout) # in case the user switched the row factory, set it to namedtuple for this query - future = self._session._create_response_future(query, parameters, trace=False, custom_payload=None) future.row_factory = named_tuple_factory future.send_request() - timeout = (max_wait - time_spent) if max_wait is not None else None try: - return future.result(timeout=timeout) + return future.result() except OperationTimedOut: raise TraceUnavailable("Trace information was not available within %f seconds" % (max_wait,)) diff --git a/docs/object_mapper.rst b/docs/object_mapper.rst index 26d78a0964..4e38994064 100644 --- a/docs/object_mapper.rst +++ b/docs/object_mapper.rst @@ -48,7 +48,7 @@ Getting Started from cassandra.cqlengine import columns from cassandra.cqlengine import connection from datetime import datetime - from cassandra.cqlengine.management import create_keyspace, sync_table + from cassandra.cqlengine.management import sync_table from cassandra.cqlengine.models import Model #first, define a model diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index e5be60759d..268e19d535 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -244,10 +244,7 @@ def test_not_implemented(self): Ensure the following methods throw NIE's. If not, come back and test them. """ c = self.make_connection() - self.assertRaises(NotImplementedError, c.close) - self.assertRaises(NotImplementedError, c.register_watcher, None, None) - self.assertRaises(NotImplementedError, c.register_watchers, None) def test_set_keyspace_blocking(self): c = self.make_connection() diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index 92351a9d1d..eea43f7586 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -47,7 +47,7 @@ def make_session(self): def make_response_future(self, session): query = SimpleStatement("SELECT * FROM foo") message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - return ResponseFuture(session, message, query) + return ResponseFuture(session, message, query, 1) def make_mock_response(self, results): return Mock(spec=ResultMessage, kind=RESULT_KIND_ROWS, results=results, paging_state=None) @@ -122,7 +122,7 @@ def test_read_timeout_error_message(self): query.retry_policy.on_read_timeout.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=ReadTimeoutErrorMessage, info={}) @@ -137,7 +137,7 @@ def test_write_timeout_error_message(self): query.retry_policy.on_write_timeout.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=WriteTimeoutErrorMessage, info={}) @@ -151,7 +151,7 @@ def test_unavailable_error_message(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=UnavailableErrorMessage, info={}) @@ -165,7 +165,7 @@ def test_retry_policy_says_ignore(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.IGNORE, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() result = Mock(spec=UnavailableErrorMessage, info={}) @@ -184,7 +184,7 @@ def test_retry_policy_says_retry(self): connection = Mock(spec=Connection) pool.borrow_connection.return_value = (connection, 1) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() rf.session._pools.get.assert_called_once_with('ip1') @@ -279,7 +279,7 @@ def test_all_pools_shutdown(self): session._load_balancer.make_query_plan.return_value = ['ip1', 'ip2'] session._pools.get.return_value.is_shutdown = True - rf = ResponseFuture(session, Mock(), Mock()) + rf = ResponseFuture(session, Mock(), Mock(), 1) rf.send_request() self.assertRaises(NoHostAvailable, rf.result) @@ -354,7 +354,7 @@ def test_errback(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() rf.add_errback(self.assertIsInstance, Exception) @@ -401,7 +401,7 @@ def test_multiple_errbacks(self): query.retry_policy.on_unavailable.return_value = (RetryPolicy.RETHROW, None) message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() callback = Mock() @@ -431,7 +431,7 @@ def test_add_callbacks(self): message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) # test errback - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() rf.add_callbacks( @@ -443,7 +443,7 @@ def test_add_callbacks(self): self.assertRaises(Exception, rf.result) # test callback - rf = ResponseFuture(session, message, query) + rf = ResponseFuture(session, message, query, 1) rf.send_request() callback = Mock() From f779da72d8374c7306ccdf81cc731af2a5323381 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 19 Jun 2015 13:01:10 -0500 Subject: [PATCH 0223/2431] For callback chaining, use old pattern of no-timeouts Avoids the overhead of timer management when using the callback chaining pattern. --- benchmarks/callback_full_pipeline.py | 2 +- cassandra/concurrent.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/callback_full_pipeline.py b/benchmarks/callback_full_pipeline.py index 707d127c9b..7baa919494 100644 --- a/benchmarks/callback_full_pipeline.py +++ b/benchmarks/callback_full_pipeline.py @@ -42,7 +42,7 @@ def insert_next(self, previous_result=sentinel): self.event.set() if next(self.num_started) <= self.num_queries: - future = self.session.execute_async(self.query, self.values) + future = self.session.execute_async(self.query, self.values, timeout=None) future.add_callbacks(self.insert_next, self.insert_next) def run(self): diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index 8d7743e1b4..b96c921bd2 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -138,7 +138,7 @@ def _handle_error(error, result_index, event, session, statements, results, return try: - future = session.execute_async(statement, params) + future = session.execute_async(statement, params, timeout=None) args = (next_index, event, session, statements, results, future, num_finished, to_execute, first_error) future.add_callbacks( callback=_execute_next, callback_args=args, @@ -176,7 +176,7 @@ def _execute_next(result, result_index, event, session, statements, results, return try: - future = session.execute_async(statement, params) + future = session.execute_async(statement, params, timeout=None) args = (next_index, event, session, statements, results, future, num_finished, to_execute, first_error) future.add_callbacks( callback=_execute_next, callback_args=args, From a327ab13a56077e007f4f74de99b950b99d66f73 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 19 Jun 2015 13:12:02 -0500 Subject: [PATCH 0224/2431] micro-optimize no timer case --- cassandra/connection.py | 38 +++++++++++++++--------------------- cassandra/io/libevreactor.py | 7 ++++--- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 95858bd8c1..20b69c117c 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -973,20 +973,22 @@ def service_timeouts(self): :return: next end time, or None """ queue = self._queue - new_timers = self._new_timers - while new_timers: - heappush(queue, new_timers.pop()) - - now = time.time() - while queue: - try: - timer = queue[0][1] - if timer.finish(now): - heappop(queue) - else: - return timer.end - except Exception: - log.exception("Exception while servicing timeout callback: ") + if self._new_timers: + new_timers = self._new_timers + while new_timers: + heappush(queue, new_timers.pop()) + + if queue: + now = time.time() + while queue: + try: + timer = queue[0][1] + if timer.finish(now): + heappop(queue) + else: + return timer.end + except Exception: + log.exception("Exception while servicing timeout callback: ") @property def next_timeout(self): @@ -994,11 +996,3 @@ def next_timeout(self): return self._queue[0][0] except IndexError: pass - - @property - def next_offset(self): - try: - next_end = self._queue[0][0] - return next_end - time.time() - except IndexError: - pass diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index af0c3a6015..9114af133b 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -19,6 +19,7 @@ import socket import ssl from threading import Lock, Thread +import time import weakref from six.moves import range @@ -144,9 +145,9 @@ def add_timer(self, timer): def _update_timer(self): if not self._shutdown: - self._timers.service_timeouts() - offset = self._timers.next_offset or 100000 # none pending; will be updated again when something new happens - self._loop_timer.start(offset) + next_end = self._timers.service_timeouts() + if next_end: + self._loop_timer.start(next_end - time.time()) # timer handles negative values else: self._loop_timer.stop() From 1ffd5190a77e59f15c1037c70ce60872b85d07b9 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 22 Jun 2015 09:55:06 -0500 Subject: [PATCH 0225/2431] For meta export, don't copy in default min/max compaction thresholds Fixes an issue where table meta export would contain min/max_threshold settings, even for compaction strategies that do not accept those. Resulting CQL would cause "ConfigurationException ... code=2300 ... Properties specified [min_threshold, max_threshold] are not understood by [some compaction strategy]" --- cassandra/metadata.py | 15 +++----- tests/integration/standard/test_metadata.py | 42 ++++++++++++++++----- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 8d550cee41..1ae5ea11ac 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1207,7 +1207,6 @@ def primary_key(self): "rows_per_partition_to_cache", "memtable_flush_period_in_ms", "populate_io_cache_on_flush", - "compaction", "compression", "default_time_to_live") @@ -1350,17 +1349,13 @@ def as_cql_query(self, formatted=False): def _make_option_strings(self): ret = [] options_copy = dict(self.options.items()) - if not options_copy.get('compaction'): - options_copy.pop('compaction', None) - actual_options = json.loads(options_copy.pop('compaction_strategy_options', '{}')) - for system_table_name, compact_option_name in self.compaction_options.items(): - value = options_copy.pop(system_table_name, None) - if value: - actual_options.setdefault(compact_option_name, value) + actual_options = json.loads(options_copy.pop('compaction_strategy_options', '{}')) + value = options_copy.pop("compaction_strategy_class", None) + actual_options.setdefault("class", value) - compaction_option_strings = ["'%s': '%s'" % (k, v) for k, v in actual_options.items()] - ret.append('compaction = {%s}' % ', '.join(compaction_option_strings)) + compaction_option_strings = ["'%s': '%s'" % (k, v) for k, v in actual_options.items()] + ret.append('compaction = {%s}' % ', '.join(compaction_option_strings)) for system_table_name in self.compaction_options.keys(): options_copy.pop(system_table_name, None) # delete if present diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index de89e8c507..611dce2307 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -313,6 +313,30 @@ def test_compression_disabled(self): tablemeta = self.get_table_metadata() self.assertIn("compression = {}", tablemeta.export_as_string()) + def test_non_size_tiered_compaction(self): + """ + test options for non-size-tiered compaction strategy + + Creates a table with LeveledCompactionStrategy, specifying one non-default option. Verifies that the option is + present in generated CQL, and that other legacy table parameters (min_threshold, max_threshold) are not included. + + @since 2.6.0 + @jira_ticket PYTHON-352 + @expected_result the options map for LeveledCompactionStrategy does not contain min_threshold, max_threshold + + @test_category metadata + """ + create_statement = self.make_create_statement(["a"], [], ["b", "c"]) + create_statement += "WITH COMPACTION = {'class': 'LeveledCompactionStrategy', 'tombstone_threshold': '0.3'}" + self.session.execute(create_statement) + + table_meta = self.get_table_metadata() + cql = table_meta.export_as_string() + self.assertIn("'tombstone_threshold': '0.3'", cql) + self.assertIn("LeveledCompactionStrategy", cql) + self.assertNotIn("min_threshold", cql) + self.assertNotIn("max_threshold", cql) + def test_refresh_schema_metadata(self): """ test for synchronously refreshing all cluster metadata @@ -670,7 +694,7 @@ def test_export_keyspace_schema_udts(self): ) WITH bloom_filter_fp_chance = 0.01 AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -691,7 +715,7 @@ def test_export_keyspace_schema_udts(self): ) WITH bloom_filter_fp_chance = 0.01 AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -926,7 +950,7 @@ def test_legacy_tables(self): AND CLUSTERING ORDER BY (t ASC, b ASC, s ASC) AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = 'Stores file meta data' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -948,7 +972,7 @@ def test_legacy_tables(self): ) WITH COMPACT STORAGE AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -967,7 +991,7 @@ def test_legacy_tables(self): ) WITH COMPACT STORAGE AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -988,7 +1012,7 @@ def test_legacy_tables(self): AND CLUSTERING ORDER BY (column1 ASC) AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -1005,7 +1029,7 @@ def test_legacy_tables(self): ) WITH COMPACT STORAGE AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -1025,7 +1049,7 @@ def test_legacy_tables(self): AND CLUSTERING ORDER BY (column1 ASC) AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = '' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 @@ -1052,7 +1076,7 @@ def test_legacy_tables(self): AND CLUSTERING ORDER BY (column1 ASC, column1 ASC, column2 ASC) AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' AND comment = 'Stores file meta data' - AND compaction = {'min_threshold': '4', 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND dclocal_read_repair_chance = 0.1 AND default_time_to_live = 0 From 22b92df9f970784968abdd4eed656a491f443c88 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 22 Jun 2015 17:11:43 -0500 Subject: [PATCH 0226/2431] Add protocol_hander_class to allow extension, serdes specialization --- cassandra/cluster.py | 18 ++- cassandra/connection.py | 12 +- cassandra/protocol.py | 190 +++++++++++++++++++------------- docs/api/cassandra/cluster.rst | 2 + docs/api/cassandra/protocol.rst | 9 ++ tests/integration/__init__.py | 2 +- 6 files changed, 151 insertions(+), 82 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 45784c4623..853a8fef85 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -60,7 +60,7 @@ IsBootstrappingErrorMessage, BatchMessage, RESULT_KIND_PREPARED, RESULT_KIND_SET_KEYSPACE, RESULT_KIND_ROWS, - RESULT_KIND_SCHEMA_CHANGE, MIN_SUPPORTED_VERSION) + RESULT_KIND_SCHEMA_CHANGE, MIN_SUPPORTED_VERSION, ProtocolHandler) from cassandra.metadata import Metadata, protect_name, murmur3 from cassandra.policies import (TokenAwarePolicy, DCAwareRoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, @@ -419,6 +419,15 @@ def auth_provider(self, value): GeventConnection will be used automatically. """ + protocol_handler_class = ProtocolHandler + """ + Specifies a protocol handler class, which can be used to override or extend features + such as message or type deserialization. + + The class must conform to the public classmethod interface defined in the default + implementation, :class:`cassandra.protocol.ProtocolHandler` + """ + control_connection_timeout = 2.0 """ A timeout, in seconds, for queries made by the control connection, such @@ -515,7 +524,8 @@ def __init__(self, idle_heartbeat_interval=30, schema_event_refresh_window=2, topology_event_refresh_window=10, - connect_timeout=5): + connect_timeout=5, + protocol_handler_class=None): """ Any of the mutable Cluster attributes may be set as keyword arguments to the constructor. @@ -559,6 +569,9 @@ def __init__(self, if connection_class is not None: self.connection_class = connection_class + if protocol_handler_class is not None: + self.protocol_handler_class = protocol_handler_class + self.metrics_enabled = metrics_enabled self.ssl_options = ssl_options self.sockopts = sockopts @@ -798,6 +811,7 @@ def _make_connection_kwargs(self, address, kwargs_dict): kwargs_dict['cql_version'] = self.cql_version kwargs_dict['protocol_version'] = self.protocol_version kwargs_dict['user_type_map'] = self._user_types + kwargs_dict['protocol_handler_class'] = self.protocol_handler_class return kwargs_dict diff --git a/cassandra/connection.py b/cassandra/connection.py index 20b69c117c..d1e3a9c97a 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -42,7 +42,7 @@ from cassandra.marshal import int32_pack, uint8_unpack from cassandra.protocol import (ReadyMessage, AuthenticateMessage, OptionsMessage, StartupMessage, ErrorMessage, CredentialsMessage, - QueryMessage, ResultMessage, decode_response, + QueryMessage, ResultMessage, ProtocolHandler, InvalidRequestException, SupportedMessage, AuthResponseMessage, AuthChallengeMessage, AuthSuccessMessage, ProtocolException, @@ -209,7 +209,7 @@ class Connection(object): def __init__(self, host='127.0.0.1', port=9042, authenticator=None, ssl_options=None, sockopts=None, compression=True, cql_version=None, protocol_version=MAX_SUPPORTED_VERSION, is_control_connection=False, - user_type_map=None): + user_type_map=None, protocol_handler_class=ProtocolHandler): self.host = host self.port = port self.authenticator = authenticator @@ -220,6 +220,8 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.protocol_version = protocol_version self.is_control_connection = is_control_connection self.user_type_map = user_type_map + self.decoder = protocol_handler_class.decode_message + self.encoder = protocol_handler_class.encode_message self._push_watchers = defaultdict(set) self._callbacks = {} self._iobuf = io.BytesIO() @@ -362,7 +364,7 @@ def send_msg(self, msg, request_id, cb): raise ConnectionShutdown("Connection to %s is closed" % self.host) self._callbacks[request_id] = cb - self.push(msg.to_binary(request_id, self.protocol_version, compression=self.compressor)) + self.push(self.encoder(msg, request_id, self.protocol_version, compressor=self.compressor)) return request_id def wait_for_response(self, msg, timeout=None): @@ -498,8 +500,8 @@ def process_msg(self, header, body): self.msg_received = True try: - response = decode_response(header.version, self.user_type_map, stream_id, - header.flags, header.opcode, body, self.decompressor) + response = self.decoder(header.version, self.user_type_map, stream_id, + header.flags, header.opcode, body, self.decompressor) except Exception as exc: log.exception("Error decoding response from Cassandra. " "opcode: %04x; message contents: %r", header.opcode, body) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 9b544236ec..0aa9ae4846 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -83,29 +83,6 @@ class _MessageType(object): custom_payload = None warnings = None - def to_binary(self, stream_id, protocol_version, compression=None): - flags = 0 - body = io.BytesIO() - if self.custom_payload: - if protocol_version < 4: - raise UnsupportedOperation("Custom key/value payloads can only be used with protocol version 4 or higher") - flags |= CUSTOM_PAYLOAD_FLAG - write_bytesmap(body, self.custom_payload) - self.send_body(body, protocol_version) - body = body.getvalue() - - if compression and len(body) > 0: - body = compression(body) - flags |= COMPRESSED_FLAG - if self.tracing: - flags |= TRACING_FLAG - - msg = io.BytesIO() - write_header(msg, protocol_version, flags, stream_id, self.opcode, len(body)) - msg.write(body) - - return msg.getvalue() - def update_custom_payload(self, other): if other: if not self.custom_payload: @@ -126,50 +103,6 @@ def _get_params(message_obj): ) -def decode_response(protocol_version, user_type_map, stream_id, flags, opcode, body, - decompressor=None): - if flags & COMPRESSED_FLAG: - if decompressor is None: - raise Exception("No de-compressor available for compressed frame!") - body = decompressor(body) - flags ^= COMPRESSED_FLAG - - body = io.BytesIO(body) - if flags & TRACING_FLAG: - trace_id = UUID(bytes=body.read(16)) - flags ^= TRACING_FLAG - else: - trace_id = None - - if flags & WARNING_FLAG: - warnings = read_stringlist(body) - flags ^= WARNING_FLAG - else: - warnings = None - - if flags & CUSTOM_PAYLOAD_FLAG: - custom_payload = read_bytesmap(body) - flags ^= CUSTOM_PAYLOAD_FLAG - else: - custom_payload = None - - if flags: - log.warning("Unknown protocol flags set: %02x. May cause problems.", flags) - - msg_class = _message_types_by_opcode[opcode] - msg = msg_class.recv_body(body, protocol_version, user_type_map) - msg.stream_id = stream_id - msg.trace_id = trace_id - msg.custom_payload = custom_payload - msg.warnings = warnings - - if msg.warnings: - for w in msg.warnings: - log.warning("Server warning: %s", w) - - return msg - - error_classes = {} @@ -609,7 +542,7 @@ class ResultMessage(_MessageType): results = None paging_state = None - _type_codes = { + type_codes = { 0x0000: CUSTOM_TYPE, 0x0001: AsciiType, 0x0002: LongType, @@ -744,7 +677,7 @@ def recv_results_schema_change(cls, f, protocol_version): def read_type(cls, f, user_type_map): optid = read_short(f) try: - typeclass = cls._type_codes[optid] + typeclass = cls.type_codes[optid] except KeyError: raise NotSupportedError("Unknown data type code 0x%04x. Have to skip" " entire result set." % (optid,)) @@ -964,13 +897,122 @@ def recv_schema_change(cls, f, protocol_version): return event -def write_header(f, version, flags, stream_id, opcode, length): +class ProtocolHandler(object): + """ + ProtocolHander handles encoding and decoding messages. + + This class can be specialized to compose Handlers which implement alternative + result decoding or type deserialization. Class definitions are passed to :class:`cassandra.cluster.Cluster` + on initialization. + + Contracted class methods are :meth:`ProtocolHandler.encode_message` and :meth:`ProtocolHandler.decode_message`. + """ + + message_types_by_opcode = _message_types_by_opcode.copy() """ - Write a CQL protocol frame header. + Default mapping of opcode to Message implementation. The default ``decode_message`` implementation uses + this to instantiate a message and populate using ``recv_body``. This mapping can be updated to inject specialized + result decoding implementations. """ - pack = v3_header_pack if version >= 3 else header_pack - f.write(pack(version, flags, stream_id, opcode)) - write_int(f, length) + + @classmethod + def encode_message(cls, msg, stream_id, protocol_version, compressor): + """ + Encodes a message using the specified frame parameters, and compressor + + :param msg: the message, typically of cassandra.protocol._MessageType, generated by the driver + :param stream_id: protocol stream id for the frame header + :param protocol_version: version for the frame header, and used encoding contents + :param compressor: optional compression function to be used on the body + :return: + """ + flags = 0 + body = io.BytesIO() + if msg.custom_payload: + if protocol_version < 4: + raise UnsupportedOperation("Custom key/value payloads can only be used with protocol version 4 or higher") + flags |= CUSTOM_PAYLOAD_FLAG + write_bytesmap(body, msg.custom_payload) + msg.send_body(body, protocol_version) + body = body.getvalue() + + if compressor and len(body) > 0: + body = compressor(body) + flags |= COMPRESSED_FLAG + + if msg.tracing: + flags |= TRACING_FLAG + + buff = io.BytesIO() + cls._write_header(buff, protocol_version, flags, stream_id, msg.opcode, len(body)) + buff.write(body) + + return buff.getvalue() + + @staticmethod + def _write_header(f, version, flags, stream_id, opcode, length): + """ + Write a CQL protocol frame header. + """ + pack = v3_header_pack if version >= 3 else header_pack + f.write(pack(version, flags, stream_id, opcode)) + write_int(f, length) + + @classmethod + def decode_message(cls, protocol_version, user_type_map, stream_id, flags, opcode, body, + decompressor): + """ + Decodes a native protocol message body + + :param protocol_version: version to use decoding contents + :param user_type_map: map[keyspace name] = map[type name] = custom type to instantiate when deserializing this type + :param stream_id: native protocol stream id from the frame header + :param flags: native protocol flags bitmap from the header + :param opcode: native protocol opcode from the header + :param body: frame body + :param decompressor: optional decompression function to inflate the body + :return: a message decoded from the body and frame attributes + """ + if flags & COMPRESSED_FLAG: + if decompressor is None: + raise Exception("No de-compressor available for compressed frame!") + body = decompressor(body) + flags ^= COMPRESSED_FLAG + + body = io.BytesIO(body) + if flags & TRACING_FLAG: + trace_id = UUID(bytes=body.read(16)) + flags ^= TRACING_FLAG + else: + trace_id = None + + if flags & WARNING_FLAG: + warnings = read_stringlist(body) + flags ^= WARNING_FLAG + else: + warnings = None + + if flags & CUSTOM_PAYLOAD_FLAG: + custom_payload = read_bytesmap(body) + flags ^= CUSTOM_PAYLOAD_FLAG + else: + custom_payload = None + + if flags: + log.warning("Unknown protocol flags set: %02x. May cause problems.", flags) + + msg_class = cls.message_types_by_opcode[opcode] + msg = msg_class.recv_body(body, protocol_version, user_type_map) + msg.stream_id = stream_id + msg.trace_id = trace_id + msg.custom_payload = custom_payload + msg.warnings = warnings + + if msg.warnings: + for w in msg.warnings: + log.warning("Server warning: %s", w) + + return msg def read_byte(f): diff --git a/docs/api/cassandra/cluster.rst b/docs/api/cassandra/cluster.rst index 3edc22c82a..ad4272c988 100644 --- a/docs/api/cassandra/cluster.rst +++ b/docs/api/cassandra/cluster.rst @@ -27,6 +27,8 @@ .. autoattribute:: connection_class + .. autoattribute:: protocol_handler_class + .. autoattribute:: metrics_enabled .. autoattribute:: metrics diff --git a/docs/api/cassandra/protocol.rst b/docs/api/cassandra/protocol.rst index 52f1287a6c..cabf2b59dd 100644 --- a/docs/api/cassandra/protocol.rst +++ b/docs/api/cassandra/protocol.rst @@ -15,3 +15,12 @@ By default these are ignored by the server. They can be useful for servers imple a custom QueryHandler. See :meth:`.Session.execute`, ::meth:`.Session.execute_async`, :attr:`.ResponseFuture.custom_payload`. + +.. autoclass:: ProtocolHandler + + .. autoattribute:: message_types_by_opcode + :annotation: = {default mapping} + + .. automethod:: encode_message + + .. automethod:: decode_message diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3568b1ffc6..057da5c80a 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -80,7 +80,7 @@ def _tuple_version(version_string): USE_CASS_EXTERNAL = bool(os.getenv('USE_CASS_EXTERNAL', False)) -default_cassandra_version = '2.1.5' +default_cassandra_version = '2.1.6' if USE_CASS_EXTERNAL: if CCMClusterFactory: From 0acb459da2f05bc65f9e3243bfaa66ef6681f6a4 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 23 Jun 2015 11:36:50 -0500 Subject: [PATCH 0227/2431] Cluster._make_connection_kwargs accept overrides for internal defaults Make CC use protocol.ProtocolHandler for internal connections, to avoid specialized implementations. --- cassandra/cluster.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 853a8fef85..c26e10f1a5 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -60,7 +60,8 @@ IsBootstrappingErrorMessage, BatchMessage, RESULT_KIND_PREPARED, RESULT_KIND_SET_KEYSPACE, RESULT_KIND_ROWS, - RESULT_KIND_SCHEMA_CHANGE, MIN_SUPPORTED_VERSION, ProtocolHandler) + RESULT_KIND_SCHEMA_CHANGE, MIN_SUPPORTED_VERSION, + ProtocolHandler) from cassandra.metadata import Metadata, protect_name, murmur3 from cassandra.policies import (TokenAwarePolicy, DCAwareRoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, @@ -802,16 +803,16 @@ def _make_connection_factory(self, host, *args, **kwargs): def _make_connection_kwargs(self, address, kwargs_dict): if self._auth_provider_callable: - kwargs_dict['authenticator'] = self._auth_provider_callable(address) + kwargs_dict.setdefault('authenticator', self._auth_provider_callable(address)) - kwargs_dict['port'] = self.port - kwargs_dict['compression'] = self.compression - kwargs_dict['sockopts'] = self.sockopts - kwargs_dict['ssl_options'] = self.ssl_options - kwargs_dict['cql_version'] = self.cql_version - kwargs_dict['protocol_version'] = self.protocol_version - kwargs_dict['user_type_map'] = self._user_types - kwargs_dict['protocol_handler_class'] = self.protocol_handler_class + kwargs_dict.setdefault('port', self.port) + kwargs_dict.setdefault('compression', self.compression) + kwargs_dict.setdefault('sockopts', self.sockopts) + kwargs_dict.setdefault('ssl_options', self.ssl_options) + kwargs_dict.setdefault('cql_version', self.cql_version) + kwargs_dict.setdefault('protocol_version', self.protocol_version) + kwargs_dict.setdefault('user_type_map', self._user_types) + kwargs_dict.setdefault('protocol_handler_class', self.protocol_handler_class) return kwargs_dict @@ -1371,7 +1372,7 @@ def _prepare_all_queries(self, host): log.debug("Preparing all known prepared statements against host %s", host) connection = None try: - connection = self.connection_factory(host.address) + connection = self.connection_factory(host.address, protocol_handler_class=ProtocolHandler) try: self.control_connection.wait_for_schema_agreement(connection) except Exception: @@ -2130,7 +2131,7 @@ def _try_connect(self, host): while True: try: - connection = self._cluster.connection_factory(host.address, is_control_connection=True) + connection = self._cluster.connection_factory(host.address, is_control_connection=True, protocol_handler_class=ProtocolHandler) break except ProtocolVersionUnsupported as e: self._cluster.protocol_downgrade(host.address, e.startup_version) From 59d4612aaccfbc56e530aff6ad4ef191c8bc0c94 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 25 Jun 2015 08:49:47 -0500 Subject: [PATCH 0228/2431] cqle: Update invalid inet test with a string that won't parse Cassandra seems to be accepting various bad strings and inserting '92.242.140.2' (for many different strings). Need to follow up on that. Meanwhile, this string does induce an Invalid request. Side note: I'm not sure this test is worth having, unless there is supposed to be some kind of local validation. We're not testing the server here. --- tests/integration/cqlengine/columns/test_validation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/cqlengine/columns/test_validation.py b/tests/integration/cqlengine/columns/test_validation.py index ce8ab987c5..8732375446 100644 --- a/tests/integration/cqlengine/columns/test_validation.py +++ b/tests/integration/cqlengine/columns/test_validation.py @@ -392,6 +392,7 @@ def test_inet_saves(self): assert m.address == "192.168.1.1" def test_non_address_fails(self): + # TODO: presently this only tests that the server blows it up. Is there supposed to be local validation? with self.assertRaises(InvalidRequest): - self.InetTestModel.create(address="ham sandwich") + self.InetTestModel.create(address="what is going on here?") From 08f768aa2230c50c546dcb877fce305740bdfe00 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 25 Jun 2015 08:52:41 -0500 Subject: [PATCH 0229/2431] Changelog and release version update. --- CHANGELOG.rst | 21 ++++++++++++++++++++- cassandra/__init__.py | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 41b79b7e54..a4111bd507 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,24 @@ +2.6.0c2 +======= +June 24, 2015 + +Features +-------- +* Automatic Protocol Version Downgrade (PYTHON-240) +* cqlengine Python 2.6 compatibility (PYTHON-288) +* Double-dollar string quote UDF body (PYTHON-345) +* Set models.DEFAULT_KEYSPACE when calling set_session (github #352) + +Bug Fixes +--------- +* Avoid stall while connecting to mixed version cluster (PYTHON-303) +* Make SSL work with AsyncoreConnection in python 2.6.9 (PYTHON-322) +* Fix Murmur3Token.from_key() on Windows (PYTHON-331) +* Fix cqlengine TimeUUID rounding error for Windows (PYTHON-341) +* Avoid invalid compaction options in CQL export for non-SizeTiered (PYTHON-352) + 2.6.0c1 -===== +======= June 4, 2015 This release adds support for Cassandra 2.2 features, including version diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 059aeb00eb..258250cf83 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 6, '0c1', 'post') +__version_info__ = (2, 6, '0c2') __version__ = '.'.join(map(str, __version_info__)) From 1d9df6cafe2e0777647d76ca2d971234c37537d6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 25 Jun 2015 09:01:25 -0500 Subject: [PATCH 0230/2431] Post release version update also removing travis status widget until the install/setup can be stabilized --- README.rst | 3 --- cassandra/__init__.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/README.rst b/README.rst index bbb63a323b..a38b771303 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,6 @@ DataStax Python Driver for Apache Cassandra =========================================== -.. image:: https://travis-ci.org/datastax/python-driver.png?branch=master - :target: https://travis-ci.org/datastax/python-driver - A modern, `feature-rich `_ and highly-tunable Python client library for Apache Cassandra (1.2+) and DataStax Enterprise (3.1+) using exclusively Cassandra's binary protocol and Cassandra Query Language v3. The driver supports Python 2.6, 2.7, 3.3, and 3.4*. diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 258250cf83..23d4623d24 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 6, '0c2') +__version_info__ = (2, 6, '0c2', 'post') __version__ = '.'.join(map(str, __version_info__)) From 0fca6e59a335bad455282f6661353348c9731b6f Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Tue, 30 Jun 2015 11:53:26 -0500 Subject: [PATCH 0231/2431] Multi-col COMPACT tables are incompatible if there are clustering cols If there are not any clustering columns, a COMPACT STORAGE table can have any number of non-PRIMARY KEY columns and still be CQL-compatible. See: https://issues.apache.org/jira/browse/CASSANDRA-9647 Done for https://datastax-oss.atlassian.net/browse/PYTHON-360 --- cassandra/metadata.py | 8 +++++-- tests/integration/standard/test_metadata.py | 25 ++++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 1ae5ea11ac..9668a8c2c4 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1227,8 +1227,12 @@ def is_cql_compatible(self): """ # no such thing as DCT in CQL incompatible = issubclass(self.comparator, types.DynamicCompositeType) - # no compact storage with more than one column beyond PK - incompatible |= self.is_compact_storage and len(self.columns) > len(self.primary_key) + 1 + + # no compact storage with more than one column beyond PK if there + # are clustering columns + incompatible |= (self.is_compact_storage and + len(self.columns) > len(self.primary_key) + 1 and + len(self.clustering_key) >= 1) return not incompatible diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 611dce2307..1d7f68df57 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -17,7 +17,9 @@ except ImportError: import unittest # noqa -import difflib, six, sys +import difflib +import six +import sys from mock import Mock from cassandra import AlreadyExists, SignatureDescriptor, UserFunctionDescriptor, UserAggregateDescriptor @@ -231,6 +233,26 @@ def test_composite_in_compound_primary_key_compact(self): self.check_create_statement(tablemeta, create_statement) + def test_cql_compatibility(self): + # having more than one non-PK column is okay if there aren't any + # clustering columns + create_statement = self.make_create_statement(["a"], [], ["b", "c", "d"], compact=True) + self.session.execute(create_statement) + tablemeta = self.get_table_metadata() + + self.assertEqual([u'a'], [c.name for c in tablemeta.partition_key]) + self.assertEqual([], tablemeta.clustering_key) + self.assertEqual([u'a', u'b', u'c', u'd'], sorted(tablemeta.columns.keys())) + + self.assertTrue(tablemeta.is_cql_compatible) + + # ... but if there are clustering columns, it's not CQL compatible. + # This is a hacky way to simulate having clustering columns. + tablemeta.clustering_key = ["foo", "bar"] + tablemeta.columns["foo"] = None + tablemeta.columns["bar"] = None + self.assertFalse(tablemeta.is_cql_compatible) + def test_compound_primary_keys_ordering(self): create_statement = self.make_create_statement(["a"], ["b"], ["c"]) create_statement += " WITH CLUSTERING ORDER BY (b DESC)" @@ -592,6 +614,7 @@ def test_refresh_user_aggregate_metadata(self): cluster2.shutdown() + class TestCodeCoverage(unittest.TestCase): def test_export_schema(self): From 0a1e472ec45b66bb6309ca753643dd192911a42f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 29 Jun 2015 15:31:28 -0500 Subject: [PATCH 0232/2431] Connection server_version property Lazily queried, set preemptively whenever a topo query is performed. --- cassandra/cluster.py | 5 +++-- cassandra/connection.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 828fa8266b..5996a70d69 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1981,7 +1981,6 @@ def __init__(self, control_connection, *args, **kwargs): self.control_connection = weakref.proxy(control_connection) def try_reconnect(self): - # we'll either get back a new Connection or a NoHostAvailable return self.control_connection._reconnect_internal() def on_reconnection(self, connection): @@ -2032,7 +2031,7 @@ class ControlConnection(object): _SELECT_TRIGGERS = "SELECT * FROM system.schema_triggers" _SELECT_PEERS = "SELECT peer, data_center, rack, tokens, rpc_address, schema_version FROM system.peers" - _SELECT_LOCAL = "SELECT cluster_name, data_center, rack, tokens, partitioner, schema_version FROM system.local WHERE key='local'" + _SELECT_LOCAL = "SELECT cluster_name, data_center, rack, tokens, partitioner, release_version, schema_version FROM system.local WHERE key='local'" _SELECT_SCHEMA_PEERS = "SELECT peer, rpc_address, schema_version FROM system.peers" _SELECT_SCHEMA_LOCAL = "SELECT schema_version FROM system.local WHERE key='local'" @@ -2448,6 +2447,8 @@ def _refresh_node_list_and_token_map(self, connection, preloaded_results=None, if partitioner and tokens: token_map[host] = tokens + connection.server_version = local_row['release_version'] + # Check metadata.partitioner to see if we haven't built anything yet. If # every node in the cluster was in the contact points, we won't discover # any new nodes, so we need this additional check. (See PYTHON-90) diff --git a/cassandra/connection.py b/cassandra/connection.py index 464ec792ba..05fc68fa54 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -196,6 +196,7 @@ class Connection(object): is_unsupported_proto_version = False is_control_connection = False + _server_version = None _iobuf = None _current_frame = None @@ -737,6 +738,18 @@ def is_idle(self): def reset_idle(self): self.msg_received = False + @property + def server_version(self): + if self._server_version is None: + query_message = QueryMessage(query="SELECT release_version FROM system.local", consistency_level=ConsistencyLevel.ONE) + message = self.wait_for_response(query_message) + self._server_version = message.results[1][0][0] # (col names, rows)[rows][first row][only item] + return self._server_version + + @server_version.setter + def server_version(self, version): + self._server_version = version + def __str__(self): status = "" if self.is_defunct: From 28f8887b992183f347bea76b3167cb8a485cca47 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 1 Jul 2015 10:58:22 -0500 Subject: [PATCH 0233/2431] Refactor schema query/build into cassandra.metadata Changes in anticipation of supporting modernized schema tables (CASSANDRA-6717) - schema change response/events normalized according to protocol spec keywords -- events propagate with intrinsic keywords - Parsing is defined by connection server version - option to specialize query/parsing by earlier server versions (not implemented -- currently just refactored current conditional logic) --- cassandra/__init__.py | 14 + cassandra/cluster.py | 243 +---- cassandra/metadata.py | 1076 ++++++++++++------- cassandra/protocol.py | 16 +- tests/integration/standard/test_metadata.py | 2 +- tests/unit/test_control_connection.py | 30 +- tests/unit/test_metadata.py | 10 +- tests/unit/test_response_future.py | 8 +- 8 files changed, 764 insertions(+), 635 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 23d4623d24..ad073f7ccc 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -126,6 +126,20 @@ def consistency_value_to_name(value): return ConsistencyLevel.value_to_name[value] if value is not None else "Not Set" +class SchemaChangeType(object): + DROPPED = 'DROPPED' + CREATED = 'CREATED' + UPDATED = 'UPDATED' + + +class SchemaTargetType(object): + KEYSPACE = 'KEYSPACE' + TABLE = 'TABLE' + TYPE = 'TYPE' + FUNCTION = 'FUNCTION' + AGGREGATE = 'AGGREGATE' + + class SignatureDescriptor(object): def __init__(self, name, type_signature): diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 5996a70d69..28d808eeca 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -44,8 +44,8 @@ from itertools import groupby from cassandra import (ConsistencyLevel, AuthenticationFailed, - InvalidRequest, OperationTimedOut, - UnsupportedOperation, Unauthorized) + OperationTimedOut, UnsupportedOperation, + SchemaTargetType) from cassandra.connection import (ConnectionException, ConnectionShutdown, ConnectionHeartbeat, ProtocolVersionUnsupported) from cassandra.cqltypes import UserType @@ -79,6 +79,7 @@ def _is_eventlet_monkey_patched(): import eventlet.patcher return eventlet.patcher.is_monkey_patched('socket') + def _is_gevent_monkey_patched(): if 'gevent.monkey' not in sys.modules: return False @@ -1201,13 +1202,28 @@ def _ensure_core_connections(self): for pool in session._pools.values(): pool.ensure_core_connections() - def _validate_refresh_schema(self, keyspace, table, usertype, function, aggregate): + @staticmethod + def _validate_refresh_schema(keyspace, table, usertype, function, aggregate): if any((table, usertype, function, aggregate)): if not keyspace: raise ValueError("keyspace is required to refresh specific sub-entity {table, usertype, function, aggregate}") if sum(1 for e in (table, usertype, function) if e) > 1: raise ValueError("{table, usertype, function, aggregate} are mutually exclusive") + @staticmethod + def _target_type_from_refresh_args(keyspace, table, usertype, function, aggregate): + if aggregate: + return SchemaTargetType.AGGREGATE + elif function: + return SchemaTargetType.FUNCTION + elif usertype: + return SchemaTargetType.TYPE + elif table: + return SchemaTargetType.TABLE + elif keyspace: + return SchemaTargetType.KEYSPACE + return None + def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, aggregate=None, max_schema_agreement_wait=None): """ .. deprecated:: 2.6.0 @@ -1242,8 +1258,10 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None log.warning(msg) self._validate_refresh_schema(keyspace, table, usertype, function, aggregate) - if not self.control_connection.refresh_schema(keyspace, table, usertype, function, - aggregate, max_schema_agreement_wait): + target_type = self._target_type_from_refresh_args(keyspace, table, usertype, function, aggregate) + if not self.control_connection.refresh_schema(target_type=target_type, keyspace=keyspace, table=table, + type=usertype, function=function, aggregate=aggregate, + schema_agreement_wait=max_schema_agreement_wait): raise Exception("Schema was not refreshed. See log for details.") def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, function=None, aggregate=None): @@ -1259,8 +1277,10 @@ def submit_schema_refresh(self, keyspace=None, table=None, usertype=None, functi log.warning(msg) self._validate_refresh_schema(keyspace, table, usertype, function, aggregate) + target_type = self._target_type_from_refresh_args(keyspace, table, usertype, function, aggregate) return self.executor.submit( - self.control_connection.refresh_schema, keyspace, table, usertype, function, aggregate) + self.control_connection.refresh_schema, target_type=target_type, keyspace=keyspace, table=table, + type=usertype, function=function, aggregate=aggregate) def refresh_schema_metadata(self, max_schema_agreement_wait=None): """ @@ -1285,7 +1305,8 @@ def refresh_keyspace_metadata(self, keyspace, max_schema_agreement_wait=None): See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ - if not self.control_connection.refresh_schema(keyspace, schema_agreement_wait=max_schema_agreement_wait): + if not self.control_connection.refresh_schema(target_type=SchemaTargetType.KEYSPACE, keyspace=keyspace, + schema_agreement_wait=max_schema_agreement_wait): raise Exception("Keyspace metadata was not refreshed. See log for details.") def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None): @@ -1295,7 +1316,7 @@ def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ - if not self.control_connection.refresh_schema(keyspace, table, schema_agreement_wait=max_schema_agreement_wait): + if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TABLE, keyspace=keyspace, table=table, schema_agreement_wait=max_schema_agreement_wait): raise Exception("Table metadata was not refreshed. See log for details.") def refresh_user_type_metadata(self, keyspace, user_type, max_schema_agreement_wait=None): @@ -1304,7 +1325,7 @@ def refresh_user_type_metadata(self, keyspace, user_type, max_schema_agreement_w See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ - if not self.control_connection.refresh_schema(keyspace, usertype=user_type, schema_agreement_wait=max_schema_agreement_wait): + if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TYPE, keyspace=keyspace, type=user_type, schema_agreement_wait=max_schema_agreement_wait): raise Exception("User Type metadata was not refreshed. See log for details.") def refresh_user_function_metadata(self, keyspace, function, max_schema_agreement_wait=None): @@ -1315,7 +1336,7 @@ def refresh_user_function_metadata(self, keyspace, function, max_schema_agreemen See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ - if not self.control_connection.refresh_schema(keyspace, function=function, schema_agreement_wait=max_schema_agreement_wait): + if not self.control_connection.refresh_schema(target_type=SchemaTargetType.FUNCTION, keyspace=keyspace, function=function, schema_agreement_wait=max_schema_agreement_wait): raise Exception("User Function metadata was not refreshed. See log for details.") def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreement_wait=None): @@ -1326,7 +1347,7 @@ def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreem See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ - if not self.control_connection.refresh_schema(keyspace, aggregate=aggregate, schema_agreement_wait=max_schema_agreement_wait): + if not self.control_connection.refresh_schema(target_type=SchemaTargetType.AGGREGATE, keyspace=keyspace, aggregate=aggregate, schema_agreement_wait=max_schema_agreement_wait): raise Exception("User Aggregate metadata was not refreshed. See log for details.") def refresh_nodes(self): @@ -2022,14 +2043,6 @@ class ControlConnection(object): Internal """ - _SELECT_KEYSPACES = "SELECT * FROM system.schema_keyspaces" - _SELECT_COLUMN_FAMILIES = "SELECT * FROM system.schema_columnfamilies" - _SELECT_COLUMNS = "SELECT * FROM system.schema_columns" - _SELECT_USERTYPES = "SELECT * FROM system.schema_usertypes" - _SELECT_FUNCTIONS = "SELECT * FROM system.schema_functions" - _SELECT_AGGREGATES = "SELECT * FROM system.schema_aggregates" - _SELECT_TRIGGERS = "SELECT * FROM system.schema_triggers" - _SELECT_PEERS = "SELECT peer, data_center, rack, tokens, rpc_address, schema_version FROM system.peers" _SELECT_LOCAL = "SELECT cluster_name, data_center, rack, tokens, partitioner, release_version, schema_version FROM system.local WHERE key='local'" @@ -2219,16 +2232,14 @@ def shutdown(self): self._connection.close() del self._connection - def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None, - aggregate=None, schema_agreement_wait=None): + def refresh_schema(self, **kwargs): if not self._meta_refresh_enabled: log.debug("[control connection] Skipping schema refresh because meta refresh is disabled") return False try: if self._connection: - return self._refresh_schema(self._connection, keyspace, table, usertype, function, - aggregate, schema_agreement_wait=schema_agreement_wait) + return self._refresh_schema(self._connection, **kwargs) except ReferenceError: pass # our weak reference to the Cluster is no good except Exception: @@ -2236,13 +2247,10 @@ def refresh_schema(self, keyspace=None, table=None, usertype=None, function=None self._signal_error() return False - def _refresh_schema(self, connection, keyspace=None, table=None, usertype=None, function=None, - aggregate=None, preloaded_results=None, schema_agreement_wait=None): + def _refresh_schema(self, connection, preloaded_results=None, schema_agreement_wait=None, **kwargs): if self._cluster.is_shutdown: return False - assert sum(1 for arg in (table, usertype, function, aggregate) if arg) <= 1 - agreed = self.wait_for_schema_agreement(connection, preloaded_results=preloaded_results, wait_time=schema_agreement_wait) @@ -2250,148 +2258,8 @@ def _refresh_schema(self, connection, keyspace=None, table=None, usertype=None, log.debug("Skipping schema refresh due to lack of schema agreement") return False - cl = ConsistencyLevel.ONE - if table: - def _handle_results(success, result): - if success: - return dict_factory(*result.results) if result else {} - else: - raise result - - # a particular table changed - where_clause = " WHERE keyspace_name = '%s' AND columnfamily_name = '%s'" % (keyspace, table) - cf_query = QueryMessage(query=self._SELECT_COLUMN_FAMILIES + where_clause, consistency_level=cl) - col_query = QueryMessage(query=self._SELECT_COLUMNS + where_clause, consistency_level=cl) - triggers_query = QueryMessage(query=self._SELECT_TRIGGERS + where_clause, consistency_level=cl) - (cf_success, cf_result), (col_success, col_result), (triggers_success, triggers_result) \ - = connection.wait_for_responses(cf_query, col_query, triggers_query, timeout=self._timeout, fail_on_error=False) - - log.debug("[control connection] Fetched table info for %s.%s, rebuilding metadata", keyspace, table) - cf_result = _handle_results(cf_success, cf_result) - col_result = _handle_results(col_success, col_result) - - # handle the triggers table not existing in Cassandra 1.2 - if not triggers_success and isinstance(triggers_result, InvalidRequest): - triggers_result = {} - else: - triggers_result = _handle_results(triggers_success, triggers_result) - - self._cluster.metadata.table_changed(keyspace, table, cf_result, col_result, triggers_result) - elif usertype: - # user defined types within this keyspace changed - where_clause = " WHERE keyspace_name = '%s' AND type_name = '%s'" % (keyspace, usertype) - types_query = QueryMessage(query=self._SELECT_USERTYPES + where_clause, consistency_level=cl) - types_result = connection.wait_for_response(types_query) - log.debug("[control connection] Fetched user type info for %s.%s, rebuilding metadata", keyspace, usertype) - types_result = dict_factory(*types_result.results) if types_result.results else {} - self._cluster.metadata.usertype_changed(keyspace, usertype, types_result) - elif function: - # user defined function within this keyspace changed - where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s' AND signature = [%s]" \ - % (keyspace, function.name, ','.join("'%s'" % t for t in function.type_signature)) - functions_query = QueryMessage(query=self._SELECT_FUNCTIONS + where_clause, consistency_level=cl) - functions_result = connection.wait_for_response(functions_query) - log.debug("[control connection] Fetched user function info for %s.%s, rebuilding metadata", keyspace, function.signature) - functions_result = dict_factory(*functions_result.results) if functions_result.results else {} - self._cluster.metadata.function_changed(keyspace, function, functions_result) - elif aggregate: - # user defined aggregate within this keyspace changed - where_clause = " WHERE keyspace_name = '%s' AND aggregate_name = '%s' AND signature = [%s]" \ - % (keyspace, aggregate.name, ','.join("'%s'" % t for t in aggregate.type_signature)) - aggregates_query = QueryMessage(query=self._SELECT_AGGREGATES + where_clause, consistency_level=cl) - aggregates_result = connection.wait_for_response(aggregates_query) - log.debug("[control connection] Fetched user aggregate info for %s.%s, rebuilding metadata", keyspace, aggregate.signature) - aggregates_result = dict_factory(*aggregates_result.results) if aggregates_result.results else {} - self._cluster.metadata.aggregate_changed(keyspace, aggregate, aggregates_result) - elif keyspace: - # only the keyspace itself changed (such as replication settings) - where_clause = " WHERE keyspace_name = '%s'" % (keyspace,) - ks_query = QueryMessage(query=self._SELECT_KEYSPACES + where_clause, consistency_level=cl) - ks_result = connection.wait_for_response(ks_query) - log.debug("[control connection] Fetched keyspace info for %s, rebuilding metadata", keyspace) - ks_result = dict_factory(*ks_result.results) if ks_result.results else {} - self._cluster.metadata.keyspace_changed(keyspace, ks_result) - else: - # build everything from scratch - queries = [ - QueryMessage(query=self._SELECT_KEYSPACES, consistency_level=cl), - QueryMessage(query=self._SELECT_COLUMN_FAMILIES, consistency_level=cl), - QueryMessage(query=self._SELECT_COLUMNS, consistency_level=cl), - QueryMessage(query=self._SELECT_USERTYPES, consistency_level=cl), - QueryMessage(query=self._SELECT_FUNCTIONS, consistency_level=cl), - QueryMessage(query=self._SELECT_AGGREGATES, consistency_level=cl), - QueryMessage(query=self._SELECT_TRIGGERS, consistency_level=cl) - ] - - responses = connection.wait_for_responses(*queries, timeout=self._timeout, fail_on_error=False) - (ks_success, ks_result), (cf_success, cf_result), \ - (col_success, col_result), (types_success, types_result), \ - (functions_success, functions_result), \ - (aggregates_success, aggregates_result), \ - (trigger_success, triggers_result) = responses - - if ks_success: - ks_result = dict_factory(*ks_result.results) - else: - raise ks_result - - if cf_success: - cf_result = dict_factory(*cf_result.results) - else: - raise cf_result - - if col_success: - col_result = dict_factory(*col_result.results) - else: - raise col_result - - # if we're connected to Cassandra < 2.0, the trigges table will not exist - if trigger_success: - triggers_result = dict_factory(*triggers_result.results) - else: - if isinstance(triggers_result, InvalidRequest): - log.debug("[control connection] triggers table not found") - triggers_result = {} - elif isinstance(triggers_result, Unauthorized): - log.warning("[control connection] this version of Cassandra does not allow access to schema_triggers metadata with authorization enabled (CASSANDRA-7967); " - "The driver will operate normally, but will not reflect triggers in the local metadata model, or schema strings.") - triggers_result = {} - else: - raise triggers_result - - # if we're connected to Cassandra < 2.1, the usertypes table will not exist - if types_success: - types_result = dict_factory(*types_result.results) if types_result.results else {} - else: - if isinstance(types_result, InvalidRequest): - log.debug("[control connection] user types table not found") - types_result = {} - else: - raise types_result - - # functions were introduced in Cassandra 2.2 - if functions_success: - functions_result = dict_factory(*functions_result.results) if functions_result.results else {} - else: - if isinstance(functions_result, InvalidRequest): - log.debug("[control connection] user functions table not found") - functions_result = {} - else: - raise functions_result - - # aggregates were introduced in Cassandra 2.2 - if aggregates_success: - aggregates_result = dict_factory(*aggregates_result.results) if aggregates_result.results else {} - else: - if isinstance(aggregates_result, InvalidRequest): - log.debug("[control connection] user aggregates table not found") - aggregates_result = {} - else: - raise aggregates_result + self._cluster.metadata.refresh(connection, self._timeout, **kwargs) - log.debug("[control connection] Fetched schema, rebuilding metadata") - self._cluster.metadata.rebuild_schema(ks_result, types_result, functions_result, - aggregates_result, cf_result, col_result, triggers_result) return True def refresh_node_list_and_token_map(self, force_token_rebuild=False): @@ -2538,13 +2406,8 @@ def _handle_status_change(self, event): def _handle_schema_change(self, event): if self._schema_event_refresh_window < 0: return - keyspace = event.get('keyspace') - table = event.get('table') - usertype = event.get('type') - function = event.get('function') - aggregate = event.get('aggregate') delay = random() * self._schema_event_refresh_window - self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype, function, aggregate) + self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, **event) def wait_for_schema_agreement(self, connection=None, preloaded_results=None, wait_time=None): @@ -2727,11 +2590,11 @@ def shutdown(self): self.is_shutdown = True self._queue.put_nowait((0, None)) - def schedule(self, delay, fn, *args): - self._insert_task(delay, (fn, args)) + def schedule(self, delay, fn, *args, **kwargs): + self._insert_task(delay, (fn, args, tuple(kwargs.items()))) - def schedule_unique(self, delay, fn, *args): - task = (fn, args) + def schedule_unique(self, delay, fn, *args, **kwargs): + task = (fn, args, tuple(kwargs.items())) if task not in self._scheduled_tasks: self._insert_task(delay, task) else: @@ -2758,8 +2621,9 @@ def run(self): return if run_at <= time.time(): self._scheduled_tasks.remove(task) - fn, args = task - future = self._executor.submit(fn, *args) + fn, args, kwargs = task + kwargs = dict(kwargs) + future = self._executor.submit(fn, *args, **kwargs) future.add_done_callback(self._log_if_failed) else: self._queue.put_nowait((run_at, task)) @@ -2777,20 +2641,18 @@ def _log_if_failed(self, future): exc_info=exc) -def refresh_schema_and_set_result(keyspace, table, usertype, function, aggregate, control_conn, response_future): +def refresh_schema_and_set_result(control_conn, response_future, **kwargs): try: if control_conn._meta_refresh_enabled: log.debug("Refreshing schema in response to schema change. " - "Keyspace: %s; Table: %s, Type: %s, Function: %s, Aggregate: %s", - keyspace, table, usertype, function, aggregate) - control_conn._refresh_schema(response_future._connection, keyspace, table, usertype, function, aggregate) + "%s", kwargs) + control_conn._refresh_schema(response_future._connection, **kwargs) else: log.debug("Skipping schema refresh in response to schema change because meta refresh is disabled; " - "Keyspace: %s; Table: %s, Type: %s, Function: %s", keyspace, table, usertype, function, aggregate) + "%s", kwargs) except Exception: log.exception("Exception refreshing schema in response to schema change:") - response_future.session.submit( - control_conn.refresh_schema, keyspace, table, usertype, function, aggregate) + response_future.session.submit(control_conn.refresh_schema, **kwargs) finally: response_future._set_final_result(None) @@ -3015,13 +2877,8 @@ def _set_result(self, response): # thread instead of the event loop thread self.session.submit( refresh_schema_and_set_result, - response.results['keyspace'], - response.results.get('table'), - response.results.get('type'), - response.results.get('function'), - response.results.get('aggregate'), self.session.cluster.control_connection, - self) + self, **response.results) else: results = getattr(response, 'results', None) if results is not None and response.kind == RESULT_KIND_ROWS: diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 9668a8c2c4..73c7687c6c 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -29,10 +29,12 @@ except ImportError as e: pass -from cassandra import SignatureDescriptor +from cassandra import SignatureDescriptor, ConsistencyLevel, InvalidRequest, SchemaChangeType, Unauthorized import cassandra.cqltypes as types from cassandra.encoder import Encoder from cassandra.marshal import varint_unpack +from cassandra.protocol import QueryMessage +from cassandra.query import dict_factory from cassandra.util import OrderedDict log = logging.getLogger(__name__) @@ -110,64 +112,38 @@ def export_schema_as_string(self): """ return "\n".join(ks.export_as_string() for ks in self.keyspaces.values()) - def rebuild_schema(self, ks_results, type_results, function_results, - aggregate_results, cf_results, col_results, triggers_result): - """ - Rebuild the view of the current schema from a fresh set of rows from - the system schema tables. - - For internal use only. - """ - cf_def_rows = defaultdict(list) - col_def_rows = defaultdict(lambda: defaultdict(list)) - usertype_rows = defaultdict(list) - fn_rows = defaultdict(list) - agg_rows = defaultdict(list) - trigger_rows = defaultdict(lambda: defaultdict(list)) - - for row in cf_results: - cf_def_rows[row["keyspace_name"]].append(row) - - for row in col_results: - ksname = row["keyspace_name"] - cfname = row["columnfamily_name"] - col_def_rows[ksname][cfname].append(row) + def refresh(self, connection, timeout, target_type=None, change_type=None, **kwargs): - for row in type_results: - usertype_rows[row["keyspace_name"]].append(row) - - for row in function_results: - fn_rows[row["keyspace_name"]].append(row) + if not target_type: + self._rebuild_all(connection, timeout) + return - for row in aggregate_results: - agg_rows[row["keyspace_name"]].append(row) + tt_lower = target_type.lower() + try: + if change_type == SchemaChangeType.DROPPED: + drop_method = getattr(self, '_drop_' + tt_lower) + drop_method(**kwargs) + else: + parser = get_schema_parser(connection, timeout) + parse_method = getattr(parser, 'get_' + tt_lower) + meta = parse_method(**kwargs) + if meta: + update_method = getattr(self, '_update_' + tt_lower) + update_method(meta) + else: + drop_method = getattr(self, '_drop_' + tt_lower) + drop_method(**kwargs) + except AttributeError: + raise ValueError("Unknown schema target_type: '%s'" % target_type) - for row in triggers_result: - ksname = row["keyspace_name"] - cfname = row["columnfamily_name"] - trigger_rows[ksname][cfname].append(row) + def _rebuild_all(self, connection, timeout): + """ + For internal use only. + """ + parser = get_schema_parser(connection, timeout) current_keyspaces = set() - for row in ks_results: - keyspace_meta = self._build_keyspace_metadata(row) - keyspace_col_rows = col_def_rows.get(keyspace_meta.name, {}) - keyspace_trigger_rows = trigger_rows.get(keyspace_meta.name, {}) - for table_row in cf_def_rows.get(keyspace_meta.name, []): - table_meta = self._build_table_metadata(keyspace_meta, table_row, keyspace_col_rows, keyspace_trigger_rows) - keyspace_meta._add_table_metadata(table_meta) - - for usertype_row in usertype_rows.get(keyspace_meta.name, []): - usertype = self._build_usertype(keyspace_meta.name, usertype_row) - keyspace_meta.user_types[usertype.name] = usertype - - for fn_row in fn_rows.get(keyspace_meta.name, []): - fn = self._build_function(keyspace_meta.name, fn_row) - keyspace_meta.functions[fn.signature] = fn - - for agg_row in agg_rows.get(keyspace_meta.name, []): - agg = self._build_aggregate(keyspace_meta.name, agg_row) - keyspace_meta.aggregates[agg.signature] = agg - + for keyspace_meta in parser.get_all_keyspaces(): current_keyspaces.add(keyspace_meta.name) old_keyspace_meta = self.keyspaces.get(keyspace_meta.name, None) self.keyspaces[keyspace_meta.name] = keyspace_meta @@ -184,16 +160,10 @@ def rebuild_schema(self, ks_results, type_results, function_results, for ksname in removed_keyspaces: self._keyspace_removed(ksname) - def keyspace_changed(self, keyspace, ks_results): - if not ks_results: - if keyspace in self.keyspaces: - del self.keyspaces[keyspace] - self._keyspace_removed(keyspace) - return - - keyspace_meta = self._build_keyspace_metadata(ks_results[0]) - old_keyspace_meta = self.keyspaces.get(keyspace, None) - self.keyspaces[keyspace] = keyspace_meta + def _update_keyspace(self, keyspace_meta): + ks_name = keyspace_meta.name + old_keyspace_meta = self.keyspaces.get(ks_name, None) + self.keyspaces[ks_name] = keyspace_meta if old_keyspace_meta: keyspace_meta.tables = old_keyspace_meta.tables keyspace_meta.user_types = old_keyspace_meta.user_types @@ -201,50 +171,69 @@ def keyspace_changed(self, keyspace, ks_results): keyspace_meta.functions = old_keyspace_meta.functions keyspace_meta.aggregates = old_keyspace_meta.aggregates if (keyspace_meta.replication_strategy != old_keyspace_meta.replication_strategy): - self._keyspace_updated(keyspace) - else: - self._keyspace_added(keyspace) - - def usertype_changed(self, keyspace, name, type_results): - if type_results: - new_usertype = self._build_usertype(keyspace, type_results[0]) - self.keyspaces[keyspace].user_types[name] = new_usertype + self._keyspace_updated(ks_name) else: - # the type was deleted - self.keyspaces[keyspace].user_types.pop(name, None) + self._keyspace_added(ks_name) - def function_changed(self, keyspace, function, function_results): - if function_results: - new_function = self._build_function(keyspace, function_results[0]) - self.keyspaces[keyspace].functions[function.signature] = new_function - else: - # the function was deleted - self.keyspaces[keyspace].functions.pop(function.signature, None) + def _drop_keyspace(self, keyspace): + if self.keyspaces.pop(keyspace, None): + self._keyspace_removed(keyspace) - def aggregate_changed(self, keyspace, aggregate, aggregate_results): - if aggregate_results: - new_aggregate = self._build_aggregate(keyspace, aggregate_results[0]) - self.keyspaces[keyspace].aggregates[aggregate.signature] = new_aggregate - else: - # the aggregate was deleted - self.keyspaces[keyspace].aggregates.pop(aggregate.signature, None) + def _update_table(self, table_meta): + try: + keyspace_meta = self.keyspaces[table_meta.keyspace_name] + table_meta.keyspace = keyspace_meta # temporary while TableMetadata.keyspace is deprecated + keyspace_meta._add_table_metadata(table_meta) + except KeyError: + # can happen if keyspace disappears while processing async event + pass - def table_changed(self, keyspace, table, cf_results, col_results, triggers_result): + def _drop_table(self, keyspace, table): try: keyspace_meta = self.keyspaces[keyspace] + keyspace_meta._drop_table_metadata(table) except KeyError: - # we're trying to update a table in a keyspace we don't know about - log.error("Tried to update schema for table '%s' in unknown keyspace '%s'", - table, keyspace) - return + # can happen if keyspace disappears while processing async event + pass - if not cf_results: - # the table was removed - keyspace_meta._drop_table_metadata(table) - else: - assert len(cf_results) == 1 - table_meta = self._build_table_metadata(keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) - keyspace_meta._add_table_metadata(table_meta) + def _update_type(self, type_meta): + try: + self.keyspaces[type_meta.keyspace].user_types[type_meta.name] = type_meta + except KeyError: + # can happen if keyspace disappears while processing async event + pass + + def _drop_type(self, keyspace, type): + try: + self.keyspaces[keyspace].user_types.pop(type, None) + except KeyError: + # can happen if keyspace disappears while processing async event + pass + + def _update_function(self, function_meta): + try: + self.keyspaces[function_meta.keyspace].functions[function_meta.signature] = function_meta + except KeyError: + # can happen if keyspace disappears while processing async event + pass + + def _drop_function(self, keyspace, function): + try: + self.keyspaces[keyspace].functions.pop(function.signature, None) + except KeyError: + pass + + def _update_aggregate(self, aggregate_meta): + try: + self.keyspaces[aggregate_meta.keyspace].aggregates[aggregate_meta.signature] = aggregate_meta + except KeyError: + pass + + def _drop_aggregate(self, keyspace, aggregate): + try: + self.keyspaces[keyspace].aggregates.pop(aggregate.signature, None) + except KeyError: + pass def _keyspace_added(self, ksname): if self.token_map: @@ -258,337 +247,118 @@ def _keyspace_removed(self, ksname): if self.token_map: self.token_map.remove_keyspace(ksname) - def _build_keyspace_metadata(self, row): - name = row["keyspace_name"] - durable_writes = row["durable_writes"] - strategy_class = row["strategy_class"] - strategy_options = json.loads(row["strategy_options"]) - return KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) + def rebuild_token_map(self, partitioner, token_map): + """ + Rebuild our view of the topology from fresh rows from the + system topology tables. + For internal use only. + """ + self.partitioner = partitioner + if partitioner.endswith('RandomPartitioner'): + token_class = MD5Token + elif partitioner.endswith('Murmur3Partitioner'): + token_class = Murmur3Token + elif partitioner.endswith('ByteOrderedPartitioner'): + token_class = BytesToken + else: + self.token_map = None + return - def _build_usertype(self, keyspace, usertype_row): - type_classes = list(map(types.lookup_casstype, usertype_row['field_types'])) - return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], - usertype_row['field_names'], type_classes) + token_to_host_owner = {} + ring = [] + for host, token_strings in six.iteritems(token_map): + for token_string in token_strings: + token = token_class(token_string) + ring.append(token) + token_to_host_owner[token] = host - def _build_function(self, keyspace, function_row): - return_type = types.lookup_casstype(function_row['return_type']) - return Function(function_row['keyspace_name'], function_row['function_name'], - function_row['signature'], function_row['argument_names'], - return_type, function_row['language'], function_row['body'], - function_row['called_on_null_input']) + all_tokens = sorted(ring) + self.token_map = TokenMap( + token_class, token_to_host_owner, all_tokens, self) - def _build_aggregate(self, keyspace, aggregate_row): - state_type = types.lookup_casstype(aggregate_row['state_type']) - initial_condition = aggregate_row['initcond'] - if initial_condition is not None: - initial_condition = state_type.deserialize(initial_condition, 3) - return_type = types.lookup_casstype(aggregate_row['return_type']) - return Aggregate(aggregate_row['keyspace_name'], aggregate_row['aggregate_name'], - aggregate_row['signature'], aggregate_row['state_func'], state_type, - aggregate_row['final_func'], initial_condition, return_type) + def get_replicas(self, keyspace, key): + """ + Returns a list of :class:`.Host` instances that are replicas for a given + partition key. + """ + t = self.token_map + if not t: + return [] + try: + return t.get_replicas(keyspace, t.token_class.from_key(key)) + except NoMurmur3: + return [] - def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): - cfname = row["columnfamily_name"] - cf_col_rows = col_rows.get(cfname, []) + def can_support_partitioner(self): + if self.partitioner.endswith('Murmur3Partitioner') and murmur3 is None: + return False + else: + return True - if not cf_col_rows: # CASSANDRA-8487 - log.warning("Building table metadata with no column meta for %s.%s", - keyspace_metadata.name, cfname) + def add_or_return_host(self, host): + """ + Returns a tuple (host, new), where ``host`` is a Host + instance, and ``new`` is a bool indicating whether + the host was newly added. + """ + with self._hosts_lock: + try: + return self._hosts[host.address], False + except KeyError: + self._hosts[host.address] = host + return host, True - comparator = types.lookup_casstype(row["comparator"]) + def remove_host(self, host): + with self._hosts_lock: + return bool(self._hosts.pop(host.address, False)) - if issubclass(comparator, types.CompositeType): - column_name_types = comparator.subtypes - is_composite_comparator = True - else: - column_name_types = (comparator,) - is_composite_comparator = False + def get_host(self, address): + return self._hosts.get(address) - num_column_name_components = len(column_name_types) - last_col = column_name_types[-1] + def all_hosts(self): + """ + Returns a list of all known :class:`.Host` instances in the cluster. + """ + with self._hosts_lock: + return self._hosts.values() - column_aliases = row.get("column_aliases", None) - clustering_rows = [r for r in cf_col_rows - if r.get('type', None) == "clustering_key"] - if len(clustering_rows) > 1: - clustering_rows = sorted(clustering_rows, key=lambda row: row.get('component_index')) +REPLICATION_STRATEGY_CLASS_PREFIX = "org.apache.cassandra.locator." - if column_aliases is not None: - column_aliases = json.loads(column_aliases) - else: - column_aliases = [r.get('column_name') for r in clustering_rows] - if is_composite_comparator: - if issubclass(last_col, types.ColumnToCollectionType): - # collections - is_compact = False - has_value = False - clustering_size = num_column_name_components - 2 - elif (len(column_aliases) == num_column_name_components - 1 - and issubclass(last_col, types.UTF8Type)): - # aliases? - is_compact = False - has_value = False - clustering_size = num_column_name_components - 1 - else: - # compact table - is_compact = True - has_value = column_aliases or not cf_col_rows - clustering_size = num_column_name_components +def trim_if_startswith(s, prefix): + if s.startswith(prefix): + return s[len(prefix):] + return s - # Some thrift tables define names in composite types (see PYTHON-192) - if not column_aliases and hasattr(comparator, 'fieldnames'): - column_aliases = comparator.fieldnames - else: - is_compact = True - if column_aliases or not cf_col_rows: - has_value = True - clustering_size = num_column_name_components - else: - has_value = False - clustering_size = 0 - table_meta = TableMetadata(keyspace_metadata, cfname) - table_meta.comparator = comparator +_replication_strategies = {} - # partition key - partition_rows = [r for r in cf_col_rows - if r.get('type', None) == "partition_key"] - if len(partition_rows) > 1: - partition_rows = sorted(partition_rows, key=lambda row: row.get('component_index')) +class ReplicationStrategyTypeType(type): + def __new__(metacls, name, bases, dct): + dct.setdefault('name', name) + cls = type.__new__(metacls, name, bases, dct) + if not name.startswith('_'): + _replication_strategies[name] = cls + return cls - key_aliases = row.get("key_aliases") - if key_aliases is not None: - key_aliases = json.loads(key_aliases) if key_aliases else [] - else: - # In 2.0+, we can use the 'type' column. In 3.0+, we have to use it. - key_aliases = [r.get('column_name') for r in partition_rows] - key_validator = row.get("key_validator") - if key_validator is not None: - key_type = types.lookup_casstype(key_validator) - key_types = key_type.subtypes if issubclass(key_type, types.CompositeType) else [key_type] - else: - key_types = [types.lookup_casstype(r.get('validator')) for r in partition_rows] +@six.add_metaclass(ReplicationStrategyTypeType) +class _ReplicationStrategy(object): + options_map = None - for i, col_type in enumerate(key_types): - if len(key_aliases) > i: - column_name = key_aliases[i] - elif i == 0: - column_name = "key" - else: - column_name = "key%d" % i + @classmethod + def create(cls, strategy_class, options_map): + if not strategy_class: + return None - col = ColumnMetadata(table_meta, column_name, col_type) - table_meta.columns[column_name] = col - table_meta.partition_key.append(col) + strategy_name = trim_if_startswith(strategy_class, REPLICATION_STRATEGY_CLASS_PREFIX) - # clustering key - for i in range(clustering_size): - if len(column_aliases) > i: - column_name = column_aliases[i] - else: - column_name = "column%d" % i - - col = ColumnMetadata(table_meta, column_name, column_name_types[i]) - table_meta.columns[column_name] = col - table_meta.clustering_key.append(col) - - # value alias (if present) - if has_value: - value_alias_rows = [r for r in cf_col_rows - if r.get('type', None) == "compact_value"] - - if not key_aliases: # TODO are we checking the right thing here? - value_alias = "value" - else: - value_alias = row.get("value_alias", None) - if value_alias is None and value_alias_rows: # CASSANDRA-8487 - # In 2.0+, we can use the 'type' column. In 3.0+, we have to use it. - value_alias = value_alias_rows[0].get('column_name') - - default_validator = row.get("default_validator") - if default_validator: - validator = types.lookup_casstype(default_validator) - else: - if value_alias_rows: # CASSANDRA-8487 - validator = types.lookup_casstype(value_alias_rows[0].get('validator')) - - col = ColumnMetadata(table_meta, value_alias, validator) - if value_alias: # CASSANDRA-8487 - table_meta.columns[value_alias] = col - - # other normal columns - for col_row in cf_col_rows: - column_meta = self._build_column_metadata(table_meta, col_row) - table_meta.columns[column_meta.name] = column_meta - - if trigger_rows: - for trigger_row in trigger_rows[cfname]: - trigger_meta = self._build_trigger_metadata(table_meta, trigger_row) - table_meta.triggers[trigger_meta.name] = trigger_meta - - table_meta.options = self._build_table_options(row) - table_meta.is_compact_storage = is_compact - - return table_meta - - def _build_table_options(self, row): - """ Setup the mostly-non-schema table options, like caching settings """ - options = dict((o, row.get(o)) for o in TableMetadata.recognized_options if o in row) - - # the option name when creating tables is "dclocal_read_repair_chance", - # but the column name in system.schema_columnfamilies is - # "local_read_repair_chance". We'll store this as dclocal_read_repair_chance, - # since that's probably what users are expecting (and we need it for the - # CREATE TABLE statement anyway). - if "local_read_repair_chance" in options: - val = options.pop("local_read_repair_chance") - options["dclocal_read_repair_chance"] = val - - return options - - def _build_column_metadata(self, table_metadata, row): - name = row["column_name"] - data_type = types.lookup_casstype(row["validator"]) - is_static = row.get("type", None) == "static" - column_meta = ColumnMetadata(table_metadata, name, data_type, is_static=is_static) - index_meta = self._build_index_metadata(column_meta, row) - column_meta.index = index_meta - if index_meta: - table_metadata.indexes[index_meta.name] = index_meta - return column_meta - - def _build_index_metadata(self, column_metadata, row): - index_name = row.get("index_name") - index_type = row.get("index_type") - if index_name or index_type: - options = row.get("index_options") - index_options = json.loads(options) if options else {} - return IndexMetadata(column_metadata, index_name, index_type, index_options) - else: - return None - - def _build_trigger_metadata(self, table_metadata, row): - name = row["trigger_name"] - options = row["trigger_options"] - trigger_meta = TriggerMetadata(table_metadata, name, options) - return trigger_meta - - def rebuild_token_map(self, partitioner, token_map): - """ - Rebuild our view of the topology from fresh rows from the - system topology tables. - For internal use only. - """ - self.partitioner = partitioner - if partitioner.endswith('RandomPartitioner'): - token_class = MD5Token - elif partitioner.endswith('Murmur3Partitioner'): - token_class = Murmur3Token - elif partitioner.endswith('ByteOrderedPartitioner'): - token_class = BytesToken - else: - self.token_map = None - return - - token_to_host_owner = {} - ring = [] - for host, token_strings in six.iteritems(token_map): - for token_string in token_strings: - token = token_class(token_string) - ring.append(token) - token_to_host_owner[token] = host - - all_tokens = sorted(ring) - self.token_map = TokenMap( - token_class, token_to_host_owner, all_tokens, self) - - def get_replicas(self, keyspace, key): - """ - Returns a list of :class:`.Host` instances that are replicas for a given - partition key. - """ - t = self.token_map - if not t: - return [] - try: - return t.get_replicas(keyspace, t.token_class.from_key(key)) - except NoMurmur3: - return [] - - def can_support_partitioner(self): - if self.partitioner.endswith('Murmur3Partitioner') and murmur3 is None: - return False - else: - return True - - def add_or_return_host(self, host): - """ - Returns a tuple (host, new), where ``host`` is a Host - instance, and ``new`` is a bool indicating whether - the host was newly added. - """ - with self._hosts_lock: - try: - return self._hosts[host.address], False - except KeyError: - self._hosts[host.address] = host - return host, True - - def remove_host(self, host): - with self._hosts_lock: - return bool(self._hosts.pop(host.address, False)) - - def get_host(self, address): - return self._hosts.get(address) - - def all_hosts(self): - """ - Returns a list of all known :class:`.Host` instances in the cluster. - """ - with self._hosts_lock: - return self._hosts.values() - - -REPLICATION_STRATEGY_CLASS_PREFIX = "org.apache.cassandra.locator." - - -def trim_if_startswith(s, prefix): - if s.startswith(prefix): - return s[len(prefix):] - return s - - -_replication_strategies = {} - - -class ReplicationStrategyTypeType(type): - def __new__(metacls, name, bases, dct): - dct.setdefault('name', name) - cls = type.__new__(metacls, name, bases, dct) - if not name.startswith('_'): - _replication_strategies[name] = cls - return cls - - -@six.add_metaclass(ReplicationStrategyTypeType) -class _ReplicationStrategy(object): - options_map = None - - @classmethod - def create(cls, strategy_class, options_map): - if not strategy_class: - return None - - strategy_name = trim_if_startswith(strategy_class, REPLICATION_STRATEGY_CLASS_PREFIX) - - rs_class = _replication_strategies.get(strategy_name, None) - if rs_class is None: - rs_class = _UnknownStrategyBuilder(strategy_name) - _replication_strategies[strategy_name] = rs_class + rs_class = _replication_strategies.get(strategy_name, None) + if rs_class is None: + rs_class = _UnknownStrategyBuilder(strategy_name) + _replication_strategies[strategy_name] = rs_class try: rs_instance = rs_class(options_map) @@ -1138,7 +908,15 @@ class TableMetadata(object): """ keyspace = None - """ An instance of :class:`~.KeyspaceMetadata`. """ + """ + An instance of :class:`~.KeyspaceMetadata`. + + .. deprecated:: 2.7.0 + + """ + + keyspace_name = None + """ String name of this Table's keyspace """ name = None """ The string name of the table. """ @@ -1236,8 +1014,8 @@ def is_cql_compatible(self): return not incompatible - def __init__(self, keyspace_metadata, name, partition_key=None, clustering_key=None, columns=None, triggers=None, options=None): - self.keyspace = keyspace_metadata + def __init__(self, keyspace_name, name, partition_key=None, clustering_key=None, columns=None, triggers=None, options=None): + self.keyspace_name = keyspace_name self.name = name self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key @@ -1258,7 +1036,7 @@ def export_as_string(self): else: # If we can't produce this table with CQL, comment inline ret = "/*\nWarning: Table %s.%s omitted because it has constructs not compatible with CQL (was created via legacy API).\n" % \ - (self.keyspace.name, self.name) + (self.keyspace_name, self.name) ret += "\nApproximate structure, for reference:\n(this should not be used to reproduce this schema)\n\n%s" % self.all_as_cql() ret += "\n*/" @@ -1283,7 +1061,7 @@ def as_cql_query(self, formatted=False): extra whitespace will be added to make the query human readable. """ ret = "CREATE TABLE %s.%s (%s" % ( - protect_name(self.keyspace.name), + protect_name(self.keyspace_name), protect_name(self.name), "\n" if formatted else "") @@ -1738,3 +1516,471 @@ def as_cql_query(self): protect_value(self.options['class']) ) return ret + + +class _SchemaParser(object): + + def __init__(self, connection, timeout): + self.connection = connection + self.timeout = timeout + + def _handle_results(self, success, result): + if success: + return dict_factory(*result.results) if result else [] + else: + raise result + + +class SchemaParserV12(_SchemaParser): + _SELECT_KEYSPACES = "SELECT * FROM system.schema_keyspaces" + _SELECT_COLUMN_FAMILIES = "SELECT * FROM system.schema_columnfamilies" + _SELECT_COLUMNS = "SELECT * FROM system.schema_columns" + + pass + + +class SchemaParserV20(SchemaParserV12): + _SELECT_TRIGGERS = "SELECT * FROM system.schema_triggers" + + +class SchemaParserV21(SchemaParserV20): + _SELECT_USERTYPES = "SELECT * FROM system.schema_usertypes" + + +class SchemaParserV22(SchemaParserV21): + _SELECT_FUNCTIONS = "SELECT * FROM system.schema_functions" + _SELECT_AGGREGATES = "SELECT * FROM system.schema_aggregates" + + def __init__(self, connection, timeout): + super(SchemaParserV22, self).__init__(connection, timeout) + self.keyspaces_result = [] + self.tables_result = [] + self.columns_result = [] + self.triggers_result = [] + self.types_result = [] + self.functions_result = [] + self.aggregates_result = [] + + self.keyspace_table_rows = defaultdict(list) + self.keyspace_table_col_rows = defaultdict(lambda: defaultdict(list)) + self.keyspace_type_rows = defaultdict(list) + self.keyspace_func_rows = defaultdict(list) + self.keyspace_agg_rows = defaultdict(list) + self.keyspace_table_trigger_rows = defaultdict(lambda: defaultdict(list)) + + def get_all_keyspaces(self): + self._query_all() + + for row in self.keyspaces_result: + keyspace_meta = self._build_keyspace_metadata(row) + + keyspace_col_rows = self.keyspace_table_col_rows.get(keyspace_meta.name, {}) + keyspace_trigger_rows = self.keyspace_table_trigger_rows.get(keyspace_meta.name, {}) + for table_row in self.keyspace_table_rows.get(keyspace_meta.name, []): + table_meta = self._build_table_metadata(keyspace_meta.name, table_row, keyspace_col_rows, keyspace_trigger_rows) + table_meta.keyspace = keyspace_meta # temporary while TableMetadata.keyspace is deprecated + keyspace_meta._add_table_metadata(table_meta) + + for usertype_row in self.keyspace_type_rows.get(keyspace_meta.name, []): + usertype = self._build_user_type(keyspace_meta.name, usertype_row) + keyspace_meta.user_types[usertype.name] = usertype + + for fn_row in self.keyspace_func_rows.get(keyspace_meta.name, []): + fn = self._build_function(keyspace_meta.name, fn_row) + keyspace_meta.functions[fn.signature] = fn + + for agg_row in self.keyspace_agg_rows.get(keyspace_meta.name, []): + agg = self._build_aggregate(keyspace_meta.name, agg_row) + keyspace_meta.aggregates[agg.signature] = agg + + yield keyspace_meta + + def get_table(self, keyspace, table): + cl = ConsistencyLevel.ONE + where_clause = " WHERE keyspace_name = '%s' AND columnfamily_name = '%s'" % (keyspace, table) + cf_query = QueryMessage(query=self._SELECT_COLUMN_FAMILIES + where_clause, consistency_level=cl) + col_query = QueryMessage(query=self._SELECT_COLUMNS + where_clause, consistency_level=cl) + triggers_query = QueryMessage(query=self._SELECT_TRIGGERS + where_clause, consistency_level=cl) + (cf_success, cf_result), (col_success, col_result), (triggers_success, triggers_result) \ + = self.connection.wait_for_responses(cf_query, col_query, triggers_query, timeout=self.timeout, fail_on_error=False) + table_result = self._handle_results(cf_success, cf_result) + col_result = self._handle_results(col_success, col_result) + + # handle the triggers table not existing in Cassandra 1.2 + if not triggers_success and isinstance(triggers_result, InvalidRequest): + triggers_result = [] + else: + triggers_result = self._handle_results(triggers_success, triggers_result) + + if table_result: + return self._build_table_metadata(keyspace, table_result[0], {table: col_result}, {table: triggers_result}) + + # TODO: refactor common query/build code + def get_type(self, keyspace, type): + where_clause = " WHERE keyspace_name = '%s' AND type_name = '%s'" % (keyspace, type) + type_query = QueryMessage(query=self._SELECT_USERTYPES + where_clause, consistency_level=ConsistencyLevel.ONE) + type_result = self.connection.wait_for_response(type_query, self.timeout) + if type_result.results: + type_result = dict_factory(*type_result.results) + return self._build_user_type(keyspace, type_result[0]) + + def get_function(self, keyspace, function): + where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s' AND signature = [%s]" \ + % (keyspace, function.name, ','.join("'%s'" % t for t in function.type_signature)) + function_query = QueryMessage(query=self._SELECT_FUNCTIONS + where_clause, consistency_level=ConsistencyLevel.ONE) + function_result = self.connection.wait_for_response(function_query, self.timeout) + if function_result.results: + function_result = dict_factory(*function_result.results) + return self._build_function(keyspace, function_result[0]) + + def get_aggregate(self, keyspace, aggregate): + # user defined aggregate within this keyspace changed + where_clause = " WHERE keyspace_name = '%s' AND aggregate_name = '%s' AND signature = [%s]" \ + % (keyspace, aggregate.name, ','.join("'%s'" % t for t in aggregate.type_signature)) + aggregate_query = QueryMessage(query=self._SELECT_AGGREGATES + where_clause, consistency_level=ConsistencyLevel.ONE) + aggregate_result = self.connection.wait_for_response(aggregate_query, self.timeout) + if aggregate_result.results: + aggregate_result = dict_factory(*aggregate_result.results) + return self._build_aggregate(keyspace, aggregate_result[0]) + + def get_keyspace(self, keyspace): + where_clause = " WHERE keyspace_name = '%s'" % (keyspace,) + ks_query = QueryMessage(query=self._SELECT_KEYSPACES + where_clause, consistency_level=ConsistencyLevel.ONE) + ks_result = self.connection.wait_for_response(ks_query, self.timeout) + if ks_result.results: + ks_result = dict_factory(*ks_result.results) + return self._build_keyspace_metadata(ks_result[0]) + + @staticmethod + def _build_keyspace_metadata(row): + name = row["keyspace_name"] + durable_writes = row["durable_writes"] + strategy_class = row["strategy_class"] + strategy_options = json.loads(row["strategy_options"]) + return KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) + + @staticmethod + def _build_user_type(keyspace, usertype_row): + type_classes = list(map(types.lookup_casstype, usertype_row['field_types'])) + return UserType(keyspace, usertype_row['type_name'], + usertype_row['field_names'], type_classes) + + @staticmethod + def _build_function(keyspace, function_row): + return_type = types.lookup_casstype(function_row['return_type']) + return Function(keyspace, function_row['function_name'], + function_row['signature'], function_row['argument_names'], + return_type, function_row['language'], function_row['body'], + function_row['called_on_null_input']) + + @staticmethod + def _build_aggregate(keyspace, aggregate_row): + state_type = types.lookup_casstype(aggregate_row['state_type']) + initial_condition = aggregate_row['initcond'] + if initial_condition is not None: + initial_condition = state_type.deserialize(initial_condition, 3) + return_type = types.lookup_casstype(aggregate_row['return_type']) + return Aggregate(keyspace, aggregate_row['aggregate_name'], + aggregate_row['signature'], aggregate_row['state_func'], state_type, + aggregate_row['final_func'], initial_condition, return_type) + + def _build_table_metadata(self, keyspace_name, row, col_rows, trigger_rows): + cfname = row["columnfamily_name"] + cf_col_rows = col_rows.get(cfname, []) + + if not cf_col_rows: # CASSANDRA-8487 + log.warning("Building table metadata with no column meta for %s.%s", + keyspace_name, cfname) + + comparator = types.lookup_casstype(row["comparator"]) + + if issubclass(comparator, types.CompositeType): + column_name_types = comparator.subtypes + is_composite_comparator = True + else: + column_name_types = (comparator,) + is_composite_comparator = False + + num_column_name_components = len(column_name_types) + last_col = column_name_types[-1] + + column_aliases = row.get("column_aliases", None) + + clustering_rows = [r for r in cf_col_rows + if r.get('type', None) == "clustering_key"] + if len(clustering_rows) > 1: + clustering_rows = sorted(clustering_rows, key=lambda row: row.get('component_index')) + + if column_aliases is not None: + column_aliases = json.loads(column_aliases) + else: + column_aliases = [r.get('column_name') for r in clustering_rows] + + if is_composite_comparator: + if issubclass(last_col, types.ColumnToCollectionType): + # collections + is_compact = False + has_value = False + clustering_size = num_column_name_components - 2 + elif (len(column_aliases) == num_column_name_components - 1 + and issubclass(last_col, types.UTF8Type)): + # aliases? + is_compact = False + has_value = False + clustering_size = num_column_name_components - 1 + else: + # compact table + is_compact = True + has_value = column_aliases or not cf_col_rows + clustering_size = num_column_name_components + + # Some thrift tables define names in composite types (see PYTHON-192) + if not column_aliases and hasattr(comparator, 'fieldnames'): + column_aliases = comparator.fieldnames + else: + is_compact = True + if column_aliases or not cf_col_rows: + has_value = True + clustering_size = num_column_name_components + else: + has_value = False + clustering_size = 0 + + table_meta = TableMetadata(keyspace_name, cfname) + table_meta.comparator = comparator + + # partition key + partition_rows = [r for r in cf_col_rows + if r.get('type', None) == "partition_key"] + + if len(partition_rows) > 1: + partition_rows = sorted(partition_rows, key=lambda row: row.get('component_index')) + + key_aliases = row.get("key_aliases") + if key_aliases is not None: + key_aliases = json.loads(key_aliases) if key_aliases else [] + else: + # In 2.0+, we can use the 'type' column. In 3.0+, we have to use it. + key_aliases = [r.get('column_name') for r in partition_rows] + + key_validator = row.get("key_validator") + if key_validator is not None: + key_type = types.lookup_casstype(key_validator) + key_types = key_type.subtypes if issubclass(key_type, types.CompositeType) else [key_type] + else: + key_types = [types.lookup_casstype(r.get('validator')) for r in partition_rows] + + for i, col_type in enumerate(key_types): + if len(key_aliases) > i: + column_name = key_aliases[i] + elif i == 0: + column_name = "key" + else: + column_name = "key%d" % i + + col = ColumnMetadata(table_meta, column_name, col_type) + table_meta.columns[column_name] = col + table_meta.partition_key.append(col) + + # clustering key + for i in range(clustering_size): + if len(column_aliases) > i: + column_name = column_aliases[i] + else: + column_name = "column%d" % i + + col = ColumnMetadata(table_meta, column_name, column_name_types[i]) + table_meta.columns[column_name] = col + table_meta.clustering_key.append(col) + + # value alias (if present) + if has_value: + value_alias_rows = [r for r in cf_col_rows + if r.get('type', None) == "compact_value"] + + if not key_aliases: # TODO are we checking the right thing here? + value_alias = "value" + else: + value_alias = row.get("value_alias", None) + if value_alias is None and value_alias_rows: # CASSANDRA-8487 + # In 2.0+, we can use the 'type' column. In 3.0+, we have to use it. + value_alias = value_alias_rows[0].get('column_name') + + default_validator = row.get("default_validator") + if default_validator: + validator = types.lookup_casstype(default_validator) + else: + if value_alias_rows: # CASSANDRA-8487 + validator = types.lookup_casstype(value_alias_rows[0].get('validator')) + + col = ColumnMetadata(table_meta, value_alias, validator) + if value_alias: # CASSANDRA-8487 + table_meta.columns[value_alias] = col + + # other normal columns + for col_row in cf_col_rows: + column_meta = self._build_column_metadata(table_meta, col_row) + table_meta.columns[column_meta.name] = column_meta + + if trigger_rows: + for trigger_row in trigger_rows[cfname]: + trigger_meta = self._build_trigger_metadata(table_meta, trigger_row) + table_meta.triggers[trigger_meta.name] = trigger_meta + + table_meta.options = self._build_table_options(row) + table_meta.is_compact_storage = is_compact + + return table_meta + + @staticmethod + def _build_table_options(row): + """ Setup the mostly-non-schema table options, like caching settings """ + options = dict((o, row.get(o)) for o in TableMetadata.recognized_options if o in row) + + # the option name when creating tables is "dclocal_read_repair_chance", + # but the column name in system.schema_columnfamilies is + # "local_read_repair_chance". We'll store this as dclocal_read_repair_chance, + # since that's probably what users are expecting (and we need it for the + # CREATE TABLE statement anyway). + if "local_read_repair_chance" in options: + val = options.pop("local_read_repair_chance") + options["dclocal_read_repair_chance"] = val + + return options + + def _build_column_metadata(self, table_metadata, row): + name = row["column_name"] + data_type = types.lookup_casstype(row["validator"]) + is_static = row.get("type", None) == "static" + column_meta = ColumnMetadata(table_metadata, name, data_type, is_static=is_static) + index_meta = self._build_index_metadata(column_meta, row) + column_meta.index = index_meta + if index_meta: + table_metadata.indexes[index_meta.name] = index_meta + return column_meta + + @staticmethod + def _build_index_metadata(column_metadata, row): + index_name = row.get("index_name") + index_type = row.get("index_type") + if index_name or index_type: + options = row.get("index_options") + index_options = json.loads(options) if options else {} + return IndexMetadata(column_metadata, index_name, index_type, index_options) + else: + return None + + @staticmethod + def _build_trigger_metadata(table_metadata, row): + name = row["trigger_name"] + options = row["trigger_options"] + trigger_meta = TriggerMetadata(table_metadata, name, options) + return trigger_meta + + def _query_all(self): + cl = ConsistencyLevel.ONE + queries = [ + QueryMessage(query=self._SELECT_KEYSPACES, consistency_level=cl), + QueryMessage(query=self._SELECT_COLUMN_FAMILIES, consistency_level=cl), + QueryMessage(query=self._SELECT_COLUMNS, consistency_level=cl), + QueryMessage(query=self._SELECT_USERTYPES, consistency_level=cl), + QueryMessage(query=self._SELECT_FUNCTIONS, consistency_level=cl), + QueryMessage(query=self._SELECT_AGGREGATES, consistency_level=cl), + QueryMessage(query=self._SELECT_TRIGGERS, consistency_level=cl) + ] + + responses = self.connection.wait_for_responses(*queries, timeout=self.timeout, fail_on_error=False) + (ks_success, ks_result), (table_success, table_result), \ + (col_success, col_result), (types_success, types_result), \ + (functions_success, functions_result), \ + (aggregates_success, aggregates_result), \ + (trigger_success, triggers_result) = responses + + self.keyspaces_result = self._handle_results(ks_success, ks_result) + self.tables_result = self._handle_results(table_success, table_result) + self.columns_result = self._handle_results(col_success, col_result) + + # if we're connected to Cassandra < 2.0, the triggers table will not exist + if trigger_success: + self.triggers_result = dict_factory(*triggers_result.results) + else: + if isinstance(triggers_result, InvalidRequest): + log.debug("triggers table not found") + elif isinstance(triggers_result, Unauthorized): + log.warning("this version of Cassandra does not allow access to schema_triggers metadata with authorization enabled (CASSANDRA-7967); " + "The driver will operate normally, but will not reflect triggers in the local metadata model, or schema strings.") + else: + raise triggers_result + + # if we're connected to Cassandra < 2.1, the usertypes table will not exist + if types_success: + self.types_result = dict_factory(*types_result.results) + else: + if isinstance(types_result, InvalidRequest): + log.debug("user types table not found") + self.types_result = {} + else: + raise types_result + + # functions were introduced in Cassandra 2.2 + if functions_success: + self.functions_result = dict_factory(*functions_result.results) + else: + if isinstance(functions_result, InvalidRequest): + log.debug("user functions table not found") + else: + raise functions_result + + # aggregates were introduced in Cassandra 2.2 + if aggregates_success: + self.aggregates_result = dict_factory(aggregates_result) + else: + if isinstance(aggregates_result, InvalidRequest): + log.debug("user aggregates table not found") + else: + raise aggregates_result + + self._aggregate_results() + + def _aggregate_results(self): + m = self.keyspace_table_rows + for row in self.tables_result: + m[row["keyspace_name"]].append(row) + + m = self.keyspace_table_col_rows + for row in self.columns_result: + ksname = row["keyspace_name"] + cfname = row["columnfamily_name"] + m[ksname][cfname].append(row) + + m = self.keyspace_type_rows + for row in self.types_result: + m[row["keyspace_name"]].append(row) + + m = self.keyspace_func_rows + for row in self.functions_result: + m[row["keyspace_name"]].append(row) + + m = self.keyspace_agg_rows + for row in self.aggregates_result: + m[row["keyspace_name"]].append(row) + + m = self.keyspace_table_trigger_rows + for row in self.triggers_result: + ksname = row["keyspace_name"] + cfname = row["columnfamily_name"] + m[ksname][cfname].append(row) + + +class SchemaParserV3(SchemaParserV22): + pass + + +def get_schema_parser(connection, timeout): + server_version = connection.server_version + if server_version.startswith('3'): + return SchemaParserV3(connection, timeout) + else: + # we could further specialize by version. Right now just refactoring the + # multi-version parser we have. + return SchemaParserV22(connection, timeout) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 9b544236ec..d257825d79 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -26,7 +26,7 @@ WriteFailure, ReadFailure, FunctionFailure, AlreadyExists, InvalidRequest, Unauthorized, UnsupportedOperation, UserFunctionDescriptor, - UserAggregateDescriptor) + UserAggregateDescriptor, SchemaTargetType) from cassandra.marshal import (int32_pack, int32_unpack, uint16_pack, uint16_unpack, int8_pack, int8_unpack, uint64_pack, header_pack, v3_header_pack) @@ -69,6 +69,7 @@ class InternalError(Exception): _UNSET_VALUE = object() + class _RegisterMessageType(type): def __init__(cls, name, bases, dct): if not name.startswith('_'): @@ -948,19 +949,22 @@ def recv_schema_change(cls, f, protocol_version): if protocol_version >= 3: target = read_string(f) keyspace = read_string(f) - event = {'change_type': change_type, 'keyspace': keyspace} - if target != "KEYSPACE": + event = {'target_type': target, 'change_type': change_type, 'keyspace': keyspace} + if target != SchemaTargetType.KEYSPACE: target_name = read_string(f) - if target == 'FUNCTION': + if target == SchemaTargetType.FUNCTION: event['function'] = UserFunctionDescriptor(target_name, [read_string(f) for _ in range(read_short(f))]) - elif target == 'AGGREGATE': + elif target == SchemaTargetType.AGGREGATE: event['aggregate'] = UserAggregateDescriptor(target_name, [read_string(f) for _ in range(read_short(f))]) else: event[target.lower()] = target_name else: keyspace = read_string(f) table = read_string(f) - event = {'change_type': change_type, 'keyspace': keyspace, 'table': table} + if table: + event = {'target_type': SchemaTargetType.TABLE, 'change_type': change_type, 'keyspace': keyspace, 'table': table} + else: + event = {'target_type': SchemaTargetType.KEYSPACE, 'change_type': change_type, 'keyspace': keyspace} return event diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 1d7f68df57..582977a95e 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -125,7 +125,7 @@ def test_basic_table_meta_properties(self): self.assertTrue(self.cfname in ksmeta.tables) tablemeta = ksmeta.tables[self.cfname] - self.assertEqual(tablemeta.keyspace, ksmeta) + self.assertEqual(tablemeta.keyspace, ksmeta) # tablemeta.keyspace is deprecated self.assertEqual(tablemeta.name, self.cfname) self.assertEqual([u'a'], [c.name for c in tablemeta.partition_key]) diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 9c93f5afe4..dc3e8fa1fc 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -20,7 +20,7 @@ from concurrent.futures import ThreadPoolExecutor from mock import Mock, ANY, call -from cassandra import OperationTimedOut +from cassandra import OperationTimedOut, SchemaTargetType, SchemaChangeType from cassandra.protocol import ResultMessage, RESULT_KIND_ROWS from cassandra.cluster import ControlConnection, _Scheduler from cassandra.pool import Host @@ -94,8 +94,8 @@ class MockConnection(object): def __init__(self): self.host = "192.168.1.0" self.local_results = [ - ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "tokens"], - [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", ["0", "100", "200"]]] + ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "release_version", "tokens"], + [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", "2.2.0", ["0", "100", "200"]]] ] self.peer_results = [ @@ -136,8 +136,8 @@ def setUp(self): def _get_matching_schema_preloaded_results(self): local_results = [ - ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "tokens"], - [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", ["0", "100", "200"]]] + ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "release_version", "tokens"], + [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", "2.2.0", ["0", "100", "200"]]] ] local_response = ResultMessage(kind=RESULT_KIND_ROWS, results=local_results) @@ -152,8 +152,8 @@ def _get_matching_schema_preloaded_results(self): def _get_nonmatching_schema_preloaded_results(self): local_results = [ - ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "tokens"], - [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", ["0", "100", "200"]]] + ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "release_version", "tokens"], + [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", "2.2.0", ["0", "100", "200"]]] ] local_response = ResultMessage(kind=RESULT_KIND_ROWS, results=local_results) @@ -402,26 +402,30 @@ def test_handle_status_change(self): def test_handle_schema_change(self): - for change_type in ('CREATED', 'DROPPED', 'UPDATED'): + change_types = [getattr(SchemaChangeType, attr) for attr in vars(SchemaChangeType) if attr[0] != '_'] + for change_type in change_types: event = { + 'target_type': SchemaTargetType.TABLE, 'change_type': change_type, 'keyspace': 'ks1', 'table': 'table1' } self.cluster.scheduler.reset_mock() self.control_connection._handle_schema_change(event) - self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', 'table1', None, None, None) + self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, **event) self.cluster.scheduler.reset_mock() - event['table'] = None + event['target_type'] = SchemaTargetType.KEYSPACE + del event['table'] self.control_connection._handle_schema_change(event) - self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', None, None, None, None) + self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, **event) def test_refresh_disabled(self): cluster = MockCluster() schema_event = { - 'change_type': 'CREATED', + 'target_type': SchemaTargetType.TABLE, + 'change_type': SchemaChangeType.CREATED, 'keyspace': 'ks1', 'table': 'table1' } @@ -463,4 +467,4 @@ def test_refresh_disabled(self): cc_no_topo_refresh._handle_schema_change(schema_event) cluster.scheduler.schedule_unique.assert_has_calls([call(ANY, cc_no_topo_refresh.refresh_node_list_and_token_map), call(0.0, cc_no_topo_refresh.refresh_schema, - schema_event['keyspace'], schema_event['table'], None, None, None)]) + **schema_event)]) diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 8bdd428c65..3da3945f2d 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -26,7 +26,7 @@ NetworkTopologyStrategy, SimpleStrategy, LocalStrategy, NoMurmur3, protect_name, protect_names, protect_value, is_valid_name, - UserType, KeyspaceMetadata, Metadata, + UserType, KeyspaceMetadata, get_schema_parser, _UnknownStrategy) from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host @@ -316,15 +316,17 @@ def test_build_index_as_cql(self): column_meta.name = 'column_name_here' column_meta.table.name = 'table_name_here' column_meta.table.keyspace.name = 'keyspace_name_here' - meta_model = Metadata() + connection = Mock() + connection.server_version = '2.1.0' + parser = get_schema_parser(connection, 0.1) row = {'index_name': 'index_name_here', 'index_type': 'index_type_here'} - index_meta = meta_model._build_index_metadata(column_meta, row) + index_meta = parser._build_index_metadata(column_meta, row) self.assertEqual(index_meta.as_cql_query(), 'CREATE INDEX index_name_here ON keyspace_name_here.table_name_here (column_name_here)') row['index_options'] = '{ "class_name": "class_name_here" }' row['index_type'] = 'CUSTOM' - index_meta = meta_model._build_index_metadata(column_meta, row) + index_meta = parser._build_index_metadata(column_meta, row) self.assertEqual(index_meta.as_cql_query(), "CREATE CUSTOM INDEX index_name_here ON keyspace_name_here.table_name_here (column_name_here) USING 'class_name_here'") diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index 92351a9d1d..600f85bcac 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -19,7 +19,7 @@ from mock import Mock, MagicMock, ANY -from cassandra import ConsistencyLevel, Unavailable +from cassandra import ConsistencyLevel, Unavailable, SchemaTargetType, SchemaChangeType from cassandra.cluster import Session, ResponseFuture, NoHostAvailable from cassandra.connection import Connection, ConnectionException from cassandra.protocol import (ReadTimeoutErrorMessage, WriteTimeoutErrorMessage, @@ -101,11 +101,13 @@ def test_schema_change_result(self): rf = self.make_response_future(session) rf.send_request() + event_results={'target_type': SchemaTargetType.TABLE, 'change_type': SchemaChangeType.CREATED, + 'keyspace': "keyspace1", "table": "table1"} result = Mock(spec=ResultMessage, kind=RESULT_KIND_SCHEMA_CHANGE, - results={'keyspace': "keyspace1", "table": "table1"}) + results=event_results) rf._set_result(result) - session.submit.assert_called_once_with(ANY, 'keyspace1', 'table1', None, None, None, ANY, rf) + session.submit.assert_called_once_with(ANY, ANY, rf, **event_results) def test_other_result_message_kind(self): session = self.make_session() From c7690aaefa5e90e71576740f2a008e183fc32513 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 1 Jul 2015 14:14:05 -0500 Subject: [PATCH 0234/2431] Don't use change_type in schema refresh. Was hoping to use change_type to avoid a query on DROPPED. However, this introduces issues when the async drop event coincides with a create or updatee on the same schema element. Always querying makes sure the refresh sees the new element, assuming schema agreement wait is enabled. --- cassandra/__init__.py | 3 +++ cassandra/metadata.py | 36 ++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index ad073f7ccc..a6beab2032 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -159,6 +159,9 @@ def signature(self): def format_signature(name, type_signature): return "%s(%s)" % (name, ','.join(t for t in type_signature)) + def __repr__(self): + return "%s(%s, %s)" % (self.__class__.__name__, self.name, self.type_signature) + class UserFunctionDescriptor(SignatureDescriptor): """ diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 73c7687c6c..b79ff10a58 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -120,19 +120,15 @@ def refresh(self, connection, timeout, target_type=None, change_type=None, **kwa tt_lower = target_type.lower() try: - if change_type == SchemaChangeType.DROPPED: + parser = get_schema_parser(connection, timeout) + parse_method = getattr(parser, 'get_' + tt_lower) + meta = parse_method(**kwargs) + if meta: + update_method = getattr(self, '_update_' + tt_lower) + update_method(meta) + else: drop_method = getattr(self, '_drop_' + tt_lower) drop_method(**kwargs) - else: - parser = get_schema_parser(connection, timeout) - parse_method = getattr(parser, 'get_' + tt_lower) - meta = parse_method(**kwargs) - if meta: - update_method = getattr(self, '_update_' + tt_lower) - update_method(meta) - else: - drop_method = getattr(self, '_drop_' + tt_lower) - drop_method(**kwargs) except AttributeError: raise ValueError("Unknown schema target_type: '%s'" % target_type) @@ -1620,8 +1616,8 @@ def get_type(self, keyspace, type): where_clause = " WHERE keyspace_name = '%s' AND type_name = '%s'" % (keyspace, type) type_query = QueryMessage(query=self._SELECT_USERTYPES + where_clause, consistency_level=ConsistencyLevel.ONE) type_result = self.connection.wait_for_response(type_query, self.timeout) - if type_result.results: - type_result = dict_factory(*type_result.results) + type_result = dict_factory(*type_result.results) + if type_result: return self._build_user_type(keyspace, type_result[0]) def get_function(self, keyspace, function): @@ -1629,8 +1625,8 @@ def get_function(self, keyspace, function): % (keyspace, function.name, ','.join("'%s'" % t for t in function.type_signature)) function_query = QueryMessage(query=self._SELECT_FUNCTIONS + where_clause, consistency_level=ConsistencyLevel.ONE) function_result = self.connection.wait_for_response(function_query, self.timeout) - if function_result.results: - function_result = dict_factory(*function_result.results) + function_result = dict_factory(*function_result.results) + if function_result: return self._build_function(keyspace, function_result[0]) def get_aggregate(self, keyspace, aggregate): @@ -1639,16 +1635,16 @@ def get_aggregate(self, keyspace, aggregate): % (keyspace, aggregate.name, ','.join("'%s'" % t for t in aggregate.type_signature)) aggregate_query = QueryMessage(query=self._SELECT_AGGREGATES + where_clause, consistency_level=ConsistencyLevel.ONE) aggregate_result = self.connection.wait_for_response(aggregate_query, self.timeout) - if aggregate_result.results: - aggregate_result = dict_factory(*aggregate_result.results) + aggregate_result = dict_factory(*aggregate_result.results) + if aggregate_result: return self._build_aggregate(keyspace, aggregate_result[0]) def get_keyspace(self, keyspace): where_clause = " WHERE keyspace_name = '%s'" % (keyspace,) ks_query = QueryMessage(query=self._SELECT_KEYSPACES + where_clause, consistency_level=ConsistencyLevel.ONE) ks_result = self.connection.wait_for_response(ks_query, self.timeout) - if ks_result.results: - ks_result = dict_factory(*ks_result.results) + ks_result = dict_factory(*ks_result.results) + if ks_result: return self._build_keyspace_metadata(ks_result[0]) @staticmethod @@ -1933,7 +1929,7 @@ def _query_all(self): # aggregates were introduced in Cassandra 2.2 if aggregates_success: - self.aggregates_result = dict_factory(aggregates_result) + self.aggregates_result = dict_factory(*aggregates_result.results) else: if isinstance(aggregates_result, InvalidRequest): log.debug("user aggregates table not found") From 483d7d366d44380c748aec01abe688e07a911f3e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 1 Jul 2015 15:00:04 -0500 Subject: [PATCH 0235/2431] Refactor common query row/build meta functionality --- cassandra/metadata.py | 58 ++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index b79ff10a58..763b081ab1 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1526,6 +1526,13 @@ def _handle_results(self, success, result): else: raise result + def _query_build_row(self, query_string, build_func): + query = QueryMessage(query=query_string, consistency_level=ConsistencyLevel.ONE) + response = self.connection.wait_for_response(query, self.timeout) + result = dict_factory(*response.results) + if result: + return build_func(result[0]) + class SchemaParserV12(_SchemaParser): _SELECT_KEYSPACES = "SELECT * FROM system.schema_keyspaces" @@ -1573,20 +1580,20 @@ def get_all_keyspaces(self): keyspace_col_rows = self.keyspace_table_col_rows.get(keyspace_meta.name, {}) keyspace_trigger_rows = self.keyspace_table_trigger_rows.get(keyspace_meta.name, {}) for table_row in self.keyspace_table_rows.get(keyspace_meta.name, []): - table_meta = self._build_table_metadata(keyspace_meta.name, table_row, keyspace_col_rows, keyspace_trigger_rows) + table_meta = self._build_table_metadata(table_row, keyspace_col_rows, keyspace_trigger_rows) table_meta.keyspace = keyspace_meta # temporary while TableMetadata.keyspace is deprecated keyspace_meta._add_table_metadata(table_meta) for usertype_row in self.keyspace_type_rows.get(keyspace_meta.name, []): - usertype = self._build_user_type(keyspace_meta.name, usertype_row) + usertype = self._build_user_type(usertype_row) keyspace_meta.user_types[usertype.name] = usertype for fn_row in self.keyspace_func_rows.get(keyspace_meta.name, []): - fn = self._build_function(keyspace_meta.name, fn_row) + fn = self._build_function(fn_row) keyspace_meta.functions[fn.signature] = fn for agg_row in self.keyspace_agg_rows.get(keyspace_meta.name, []): - agg = self._build_aggregate(keyspace_meta.name, agg_row) + agg = self._build_aggregate(agg_row) keyspace_meta.aggregates[agg.signature] = agg yield keyspace_meta @@ -1609,43 +1616,25 @@ def get_table(self, keyspace, table): triggers_result = self._handle_results(triggers_success, triggers_result) if table_result: - return self._build_table_metadata(keyspace, table_result[0], {table: col_result}, {table: triggers_result}) + return self._build_table_metadata(table_result[0], {table: col_result}, {table: triggers_result}) - # TODO: refactor common query/build code def get_type(self, keyspace, type): where_clause = " WHERE keyspace_name = '%s' AND type_name = '%s'" % (keyspace, type) - type_query = QueryMessage(query=self._SELECT_USERTYPES + where_clause, consistency_level=ConsistencyLevel.ONE) - type_result = self.connection.wait_for_response(type_query, self.timeout) - type_result = dict_factory(*type_result.results) - if type_result: - return self._build_user_type(keyspace, type_result[0]) + return self._query_build_row(self._SELECT_USERTYPES + where_clause, self._build_user_type) def get_function(self, keyspace, function): where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s' AND signature = [%s]" \ % (keyspace, function.name, ','.join("'%s'" % t for t in function.type_signature)) - function_query = QueryMessage(query=self._SELECT_FUNCTIONS + where_clause, consistency_level=ConsistencyLevel.ONE) - function_result = self.connection.wait_for_response(function_query, self.timeout) - function_result = dict_factory(*function_result.results) - if function_result: - return self._build_function(keyspace, function_result[0]) + return self._query_build_row(self._SELECT_FUNCTIONS + where_clause, self._build_function) def get_aggregate(self, keyspace, aggregate): - # user defined aggregate within this keyspace changed where_clause = " WHERE keyspace_name = '%s' AND aggregate_name = '%s' AND signature = [%s]" \ % (keyspace, aggregate.name, ','.join("'%s'" % t for t in aggregate.type_signature)) - aggregate_query = QueryMessage(query=self._SELECT_AGGREGATES + where_clause, consistency_level=ConsistencyLevel.ONE) - aggregate_result = self.connection.wait_for_response(aggregate_query, self.timeout) - aggregate_result = dict_factory(*aggregate_result.results) - if aggregate_result: - return self._build_aggregate(keyspace, aggregate_result[0]) + return self._query_build_row(self._SELECT_AGGREGATES + where_clause, self._build_aggregate) def get_keyspace(self, keyspace): where_clause = " WHERE keyspace_name = '%s'" % (keyspace,) - ks_query = QueryMessage(query=self._SELECT_KEYSPACES + where_clause, consistency_level=ConsistencyLevel.ONE) - ks_result = self.connection.wait_for_response(ks_query, self.timeout) - ks_result = dict_factory(*ks_result.results) - if ks_result: - return self._build_keyspace_metadata(ks_result[0]) + return self._query_build_row(self._SELECT_KEYSPACES + where_clause, self._build_keyspace_metadata) @staticmethod def _build_keyspace_metadata(row): @@ -1656,31 +1645,32 @@ def _build_keyspace_metadata(row): return KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) @staticmethod - def _build_user_type(keyspace, usertype_row): + def _build_user_type(usertype_row): type_classes = list(map(types.lookup_casstype, usertype_row['field_types'])) - return UserType(keyspace, usertype_row['type_name'], + return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], usertype_row['field_names'], type_classes) @staticmethod - def _build_function(keyspace, function_row): + def _build_function(function_row): return_type = types.lookup_casstype(function_row['return_type']) - return Function(keyspace, function_row['function_name'], + return Function(function_row['keyspace_name'], function_row['function_name'], function_row['signature'], function_row['argument_names'], return_type, function_row['language'], function_row['body'], function_row['called_on_null_input']) @staticmethod - def _build_aggregate(keyspace, aggregate_row): + def _build_aggregate(aggregate_row): state_type = types.lookup_casstype(aggregate_row['state_type']) initial_condition = aggregate_row['initcond'] if initial_condition is not None: initial_condition = state_type.deserialize(initial_condition, 3) return_type = types.lookup_casstype(aggregate_row['return_type']) - return Aggregate(keyspace, aggregate_row['aggregate_name'], + return Aggregate(aggregate_row['keyspace_name'], aggregate_row['aggregate_name'], aggregate_row['signature'], aggregate_row['state_func'], state_type, aggregate_row['final_func'], initial_condition, return_type) - def _build_table_metadata(self, keyspace_name, row, col_rows, trigger_rows): + def _build_table_metadata(self, row, col_rows, trigger_rows): + keyspace_name = row["keyspace_name"] cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) From 179d0b6503028e54af6b4b5da509cce2e04fe21a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 1 Jul 2015 16:44:41 -0500 Subject: [PATCH 0236/2431] integration test setup: use cluster meta instead of selecting directly. Required for version-dependent schema parsing. --- tests/integration/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3568b1ffc6..33d3ae93c0 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -280,10 +280,8 @@ def setup_keyspace(ipformat=None): session = cluster.connect() try: - results = execute_until_pass(session, "SELECT keyspace_name FROM system.schema_keyspaces") - existing_keyspaces = [row[0] for row in results] for ksname in ('test1rf', 'test2rf', 'test3rf'): - if ksname in existing_keyspaces: + if ksname in cluster.metadata.keyspaces: execute_until_pass(session, "DROP KEYSPACE %s" % ksname) ddl = ''' From 5c585e2304e9d987528742e88b999176623be1cd Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 1 Jul 2015 16:45:46 -0500 Subject: [PATCH 0237/2431] Initial SchemaParser for C* 3 - select from new 'system_schema' tables - use new keyspaces 'replication' config map --- cassandra/metadata.py | 49 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 763b081ab1..6b177d7049 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1534,26 +1534,17 @@ def _query_build_row(self, query_string, build_func): return build_func(result[0]) -class SchemaParserV12(_SchemaParser): +class SchemaParserV22(_SchemaParser): _SELECT_KEYSPACES = "SELECT * FROM system.schema_keyspaces" _SELECT_COLUMN_FAMILIES = "SELECT * FROM system.schema_columnfamilies" _SELECT_COLUMNS = "SELECT * FROM system.schema_columns" - - pass - - -class SchemaParserV20(SchemaParserV12): _SELECT_TRIGGERS = "SELECT * FROM system.schema_triggers" - - -class SchemaParserV21(SchemaParserV20): - _SELECT_USERTYPES = "SELECT * FROM system.schema_usertypes" - - -class SchemaParserV22(SchemaParserV21): + _SELECT_TYPES = "SELECT * FROM system.schema_usertypes" _SELECT_FUNCTIONS = "SELECT * FROM system.schema_functions" _SELECT_AGGREGATES = "SELECT * FROM system.schema_aggregates" + _table_name_col = 'columnfamily_name' + def __init__(self, connection, timeout): super(SchemaParserV22, self).__init__(connection, timeout) self.keyspaces_result = [] @@ -1600,7 +1591,7 @@ def get_all_keyspaces(self): def get_table(self, keyspace, table): cl = ConsistencyLevel.ONE - where_clause = " WHERE keyspace_name = '%s' AND columnfamily_name = '%s'" % (keyspace, table) + where_clause = " WHERE keyspace_name = '%s' AND %s = '%s'" % (keyspace, self._table_name_col, table) cf_query = QueryMessage(query=self._SELECT_COLUMN_FAMILIES + where_clause, consistency_level=cl) col_query = QueryMessage(query=self._SELECT_COLUMNS + where_clause, consistency_level=cl) triggers_query = QueryMessage(query=self._SELECT_TRIGGERS + where_clause, consistency_level=cl) @@ -1620,7 +1611,7 @@ def get_table(self, keyspace, table): def get_type(self, keyspace, type): where_clause = " WHERE keyspace_name = '%s' AND type_name = '%s'" % (keyspace, type) - return self._query_build_row(self._SELECT_USERTYPES + where_clause, self._build_user_type) + return self._query_build_row(self._SELECT_TYPES + where_clause, self._build_user_type) def get_function(self, keyspace, function): where_clause = " WHERE keyspace_name = '%s' AND function_name = '%s' AND signature = [%s]" \ @@ -1671,7 +1662,7 @@ def _build_aggregate(aggregate_row): def _build_table_metadata(self, row, col_rows, trigger_rows): keyspace_name = row["keyspace_name"] - cfname = row["columnfamily_name"] + cfname = row[self._table_name_col] cf_col_rows = col_rows.get(cfname, []) if not cf_col_rows: # CASSANDRA-8487 @@ -1869,7 +1860,7 @@ def _query_all(self): QueryMessage(query=self._SELECT_KEYSPACES, consistency_level=cl), QueryMessage(query=self._SELECT_COLUMN_FAMILIES, consistency_level=cl), QueryMessage(query=self._SELECT_COLUMNS, consistency_level=cl), - QueryMessage(query=self._SELECT_USERTYPES, consistency_level=cl), + QueryMessage(query=self._SELECT_TYPES, consistency_level=cl), QueryMessage(query=self._SELECT_FUNCTIONS, consistency_level=cl), QueryMessage(query=self._SELECT_AGGREGATES, consistency_level=cl), QueryMessage(query=self._SELECT_TRIGGERS, consistency_level=cl) @@ -1936,7 +1927,7 @@ def _aggregate_results(self): m = self.keyspace_table_col_rows for row in self.columns_result: ksname = row["keyspace_name"] - cfname = row["columnfamily_name"] + cfname = row[self._table_name_col] m[ksname][cfname].append(row) m = self.keyspace_type_rows @@ -1954,12 +1945,28 @@ def _aggregate_results(self): m = self.keyspace_table_trigger_rows for row in self.triggers_result: ksname = row["keyspace_name"] - cfname = row["columnfamily_name"] + cfname = row[self._table_name_col] m[ksname][cfname].append(row) class SchemaParserV3(SchemaParserV22): - pass + _SELECT_KEYSPACES = "SELECT * FROM system_schema.keyspaces" + _SELECT_COLUMN_FAMILIES = "SELECT * FROM system_schema.tables" + _SELECT_COLUMNS = "SELECT * FROM system_schema.columns" + _SELECT_TRIGGERS = "SELECT * FROM system_schema.triggers" + _SELECT_TYPES = "SELECT * FROM system_schema.types" + _SELECT_FUNCTIONS = "SELECT * FROM system_schema.functions" + _SELECT_AGGREGATES = "SELECT * FROM system_schema.aggregates" + + _table_name_col = 'table_name' + + @staticmethod + def _build_keyspace_metadata(row): + name = row["keyspace_name"] + durable_writes = row["durable_writes"] + strategy_options = dict(row["replication"]) + strategy_class = strategy_options.pop("class") + return KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) def get_schema_parser(connection, timeout): @@ -1968,5 +1975,5 @@ def get_schema_parser(connection, timeout): return SchemaParserV3(connection, timeout) else: # we could further specialize by version. Right now just refactoring the - # multi-version parser we have. + # multi-version parser we have as of C* 2.2.0rc1. return SchemaParserV22(connection, timeout) From 5eb47f3dd2957ca50649d70fac85f703146dae29 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 7 Jul 2015 13:52:19 -0500 Subject: [PATCH 0238/2431] format strings for Python 2.6 in test_udt integration test --- tests/integration/standard/test_udts.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/integration/standard/test_udts.py b/tests/integration/standard/test_udts.py index eeef0508ba..495b67575d 100644 --- a/tests/integration/standard/test_udts.py +++ b/tests/integration/standard/test_udts.py @@ -295,8 +295,8 @@ def test_can_insert_udts_with_varying_lengths(self): MAX_TEST_LENGTH = 1024 # create the seed udt, increase timeout to avoid the query failure on slow systems - s.execute("CREATE TYPE lengthy_udt ({})" - .format(', '.join(['v_{} int'.format(i) + s.execute("CREATE TYPE lengthy_udt ({0})" + .format(', '.join(['v_{0} int'.format(i) for i in range(MAX_TEST_LENGTH)]))) # create a table with multiple sizes of nested udts @@ -306,7 +306,7 @@ def test_can_insert_udts_with_varying_lengths(self): "v frozen)") # create and register the seed udt type - udt = namedtuple('lengthy_udt', tuple(['v_{}'.format(i) for i in range(MAX_TEST_LENGTH)])) + udt = namedtuple('lengthy_udt', tuple(['v_{0}'.format(i) for i in range(MAX_TEST_LENGTH)])) c.register_user_type("udttests", "lengthy_udt", udt) # verify inserts and reads @@ -330,7 +330,7 @@ def nested_udt_schema_helper(self, session, MAX_NESTING_DEPTH): # create the nested udts for i in range(MAX_NESTING_DEPTH): - execute_until_pass(session, "CREATE TYPE depth_{} (value frozen)".format(i + 1, i)) + execute_until_pass(session, "CREATE TYPE depth_{0} (value frozen)".format(i + 1, i)) # create a table with multiple sizes of nested udts # no need for all nested types, only a spot checked few and the largest one @@ -390,9 +390,9 @@ def test_can_insert_nested_registered_udts(self): # create and register the nested udt types for i in range(MAX_NESTING_DEPTH): - udt = namedtuple('depth_{}'.format(i + 1), ('value')) + udt = namedtuple('depth_{0}'.format(i + 1), ('value')) udts.append(udt) - c.register_user_type("udttests", "depth_{}".format(i + 1), udts[i + 1]) + c.register_user_type("udttests", "depth_{0}".format(i + 1), udts[i + 1]) # insert udts and verify inserts with reads self.nested_udt_verification_helper(s, MAX_NESTING_DEPTH, udts) @@ -420,7 +420,7 @@ def test_can_insert_nested_unregistered_udts(self): # create the nested udt types for i in range(MAX_NESTING_DEPTH): - udt = namedtuple('depth_{}'.format(i + 1), ('value')) + udt = namedtuple('depth_{0}'.format(i + 1), ('value')) udts.append(udt) # insert udts via prepared statements and verify inserts with reads @@ -461,9 +461,9 @@ def test_can_insert_nested_registered_udts_with_different_namedtuples(self): # create and register the nested udt types for i in range(MAX_NESTING_DEPTH): - udt = namedtuple('level_{}'.format(i + 1), ('value')) + udt = namedtuple('level_{0}'.format(i + 1), ('value')) udts.append(udt) - c.register_user_type("udttests", "depth_{}".format(i + 1), udts[i + 1]) + c.register_user_type("udttests", "depth_{0}".format(i + 1), udts[i + 1]) # insert udts and verify inserts with reads self.nested_udt_verification_helper(s, MAX_NESTING_DEPTH, udts) @@ -514,7 +514,7 @@ def test_can_insert_udt_all_datatypes(self): # register UDT alphabet_list = [] for i in range(ord('a'), ord('a') + len(PRIMITIVE_DATATYPES)): - alphabet_list.append('{}'.format(chr(i))) + alphabet_list.append('{0}'.format(chr(i))) Alldatatypes = namedtuple("alldatatypes", alphabet_list) c.register_user_type("udttests", "alldatatypes", Alldatatypes) @@ -651,4 +651,4 @@ def test_can_insert_nested_collections(self): key2 = nested_collection_udt_nested({3: 'v3'}, value.t, value.l, value.s, value) validate('map_udt', OrderedMap([(key, value), (key2, value)])) - c.shutdown() \ No newline at end of file + c.shutdown() From a15dac726feb4e6c9b5ba2f1f387a9bcb46c267b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 7 Jul 2015 15:08:43 -0500 Subject: [PATCH 0239/2431] Make integration test_udts work in Python3 --- tests/integration/standard/test_udts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/standard/test_udts.py b/tests/integration/standard/test_udts.py index 495b67575d..72aa658165 100644 --- a/tests/integration/standard/test_udts.py +++ b/tests/integration/standard/test_udts.py @@ -19,6 +19,7 @@ from collections import namedtuple from functools import partial +import six from cassandra import InvalidRequest from cassandra.cluster import Cluster, UserTypeDoesNotExist @@ -277,10 +278,9 @@ def test_can_insert_udts_with_nulls(self): self.assertEqual((None, None, None, None), s.execute(select)[0].b) # also test empty strings - s.execute(insert, [User('', None, None, '')]) + s.execute(insert, [User('', None, None, six.binary_type())]) results = s.execute("SELECT b FROM mytable WHERE a=0") - self.assertEqual(('', None, None, ''), results[0].b) - self.assertEqual(('', None, None, ''), s.execute(select)[0].b) + self.assertEqual(('', None, None, six.binary_type()), results[0].b) c.shutdown() @@ -292,7 +292,7 @@ def test_can_insert_udts_with_varying_lengths(self): c = Cluster(protocol_version=PROTOCOL_VERSION) s = c.connect("udttests") - MAX_TEST_LENGTH = 1024 + MAX_TEST_LENGTH = 254 # create the seed udt, increase timeout to avoid the query failure on slow systems s.execute("CREATE TYPE lengthy_udt ({0})" From 3cb1b603af7c538132315ca12bd82f2b9ea67a7a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 7 Jul 2015 15:36:55 -0500 Subject: [PATCH 0240/2431] Make integration test_custom_payload work in Python3 --- .../integration/standard/test_custom_payload.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/integration/standard/test_custom_payload.py b/tests/integration/standard/test_custom_payload.py index 87b38fac40..04b0b1f739 100644 --- a/tests/integration/standard/test_custom_payload.py +++ b/tests/integration/standard/test_custom_payload.py @@ -18,6 +18,8 @@ except ImportError: import unittest +import six + from cassandra.query import (SimpleStatement, BatchStatement, BatchType) from cassandra.cluster import Cluster @@ -123,29 +125,29 @@ def validate_various_custom_payloads(self, statement): """ # Simple key value - custom_payload = {'test': 'test_return'} + custom_payload = {'test': b'test_return'} self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) # no key value - custom_payload = {'': ''} + custom_payload = {'': b''} self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) # Space value - custom_payload = {' ': ' '} + custom_payload = {' ': b' '} self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) # Long key value pair - key_value = "x" * 10000 - custom_payload = {key_value: key_value} + key_value = "x" * 10 + custom_payload = {key_value: six.b(key_value)} self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) # Max supported value key pairs according C* binary protocol v4 should be 65534 (unsigned short max value) for i in range(65534): - custom_payload[str(i)] = str(i) + custom_payload[str(i)] = six.b('x') self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) # Add one custom payload to this is too many key value pairs and should fail - custom_payload[str(65535)] = str(65535) + custom_payload[str(65535)] = six.b('x') with self.assertRaises(ValueError): self.execute_async_validate_custom_payload(statement=statement, custom_payload=custom_payload) From 858a26879c8f8ae2b3d7dba44fe4d5e5bbbd8501 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 7 Jul 2015 16:59:31 -0500 Subject: [PATCH 0241/2431] Mkae integration test_client_warnings work in Python3 --- tests/integration/standard/test_client_warnings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/standard/test_client_warnings.py b/tests/integration/standard/test_client_warnings.py index 173cc69756..14405a8df1 100644 --- a/tests/integration/standard/test_client_warnings.py +++ b/tests/integration/standard/test_client_warnings.py @@ -47,7 +47,6 @@ def setUpClass(cls): for x in range(213): cls.warn_batch.add(cls.prepared, (x, x, 1)) - @classmethod def tearDownClass(cls): if PROTOCOL_VERSION < 4: @@ -105,7 +104,7 @@ def test_warning_with_custom_payload(self): - batch_size_warn_threshold_in_kb: 5 @test_category queries:client_warning """ - payload = {'key': 'value'} + payload = {'key': b'value'} future = self.session.execute_async(self.warn_batch, custom_payload=payload) future.result() self.assertEqual(len(future.warnings), 1) @@ -123,7 +122,7 @@ def test_warning_with_trace_and_custom_payload(self): - batch_size_warn_threshold_in_kb: 5 @test_category queries:client_warning """ - payload = {'key': 'value'} + payload = {'key': b'value'} future = self.session.execute_async(self.warn_batch, trace=True, custom_payload=payload) future.result() self.assertEqual(len(future.warnings), 1) From 9de29a76650c4198a86fe09a913d6594ff1f03af Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Jul 2015 09:46:00 -0500 Subject: [PATCH 0242/2431] Add optional cythonized core driver files. --- .gitignore | 3 +++ cassandra/marshal.py | 6 ++++-- setup.py | 28 ++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 664c6b45ef..ee93232c33 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ tests/integration/ccm setuptools*.tar.gz setuptools*.egg +cassandra/*.c +!cassandra/murmur3.c + # OSX .DS_Store diff --git a/cassandra/marshal.py b/cassandra/marshal.py index 6451ab00ae..fba5b78944 100644 --- a/cassandra/marshal.py +++ b/cassandra/marshal.py @@ -48,13 +48,15 @@ def _make_packer(format_string): def varint_unpack(term): val = int(''.join("%02x" % i for i in term), 16) if (term[0] & 128) != 0: - val -= 1 << (len(term) * 8) + len_term = len(term) # pulling this out of the expression to avoid overflow in cython optimized code + val -= 1 << (len_term * 8) return val else: def varint_unpack(term): # noqa val = int(term.encode('hex'), 16) if (ord(term[0]) & 128) != 0: - val = val - (1 << (len(term) * 8)) + len_term = len(term) # pulling this out of the expression to avoid overflow in cython optimized code + val = val - (1 << (len_term * 8)) return val diff --git a/setup.py b/setup.py index e85d332d37..b375458153 100644 --- a/setup.py +++ b/setup.py @@ -234,16 +234,28 @@ def run_setup(extensions): ], **kw) -extensions = [murmur3_ext, libev_ext] +extensions = [] + +if "--no-murmur3" not in sys.argv: + extensions.append(murmur3_ext) + +if "--no-libev" not in sys.argv: + extensions.append(libev_ext) + +if "--no-cython" not in sys.argv: + try: + from Cython.Build import cythonize + cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'marshal', 'metadata', 'pool', 'protocol', 'query', 'util'] + extensions.extend(cythonize( + [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], extra_compile_args=['-Wno-unused-function']) for m in cython_candidates], + exclude_failures=True)) + except ImportError: + warnings.warn("Cython is not installed. Not compiling core driver files as extensions (optional).") + if "--no-extensions" in sys.argv: - sys.argv = [a for a in sys.argv if a != "--no-extensions"] extensions = [] -elif "--no-murmur3" in sys.argv: - sys.argv = [a for a in sys.argv if a != "--no-murmur3"] - extensions.remove(murmur3_ext) -elif "--no-libev" in sys.argv: - sys.argv = [a for a in sys.argv if a != "--no-libev"] - extensions.remove(libev_ext) + +sys.argv = [a for a in sys.argv if a not in ("--no-murmur3", "--no-libev", "--no-cython", "--no-extensions")] is_windows = os.name == 'nt' if is_windows: From d14c627a7683f56fdd305e02b737c9df3b9827ef Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 8 Jul 2015 11:56:59 -0500 Subject: [PATCH 0243/2431] Reorganize conditional extensions for windows also, don't pass gcc-specific compile options when building cython modules for windows --- setup.py | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index b375458153..b5a08ab21c 100644 --- a/setup.py +++ b/setup.py @@ -234,20 +234,37 @@ def run_setup(extensions): ], **kw) +is_windows = os.name == 'nt' +if is_windows: + build_extensions.error_message = """ +=============================================================================== +WARNING: could not compile %s. + +The C extensions are not required for the driver to run, but they add support +for token-aware routing with the Murmur3Partitioner. + +On Windows, make sure Visual Studio or an SDK is installed, and your environment +is configured to build for the appropriate architecture (matching your Python runtime). +This is often a matter of using vcvarsall.bat from your install directory, or running +from a command prompt in the Visual Studio Tools Start Menu. +=============================================================================== +""" + extensions = [] if "--no-murmur3" not in sys.argv: extensions.append(murmur3_ext) -if "--no-libev" not in sys.argv: +if "--no-libev" not in sys.argv and not is_windows: extensions.append(libev_ext) if "--no-cython" not in sys.argv: try: from Cython.Build import cythonize cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'marshal', 'metadata', 'pool', 'protocol', 'query', 'util'] + compile_args = [] if is_windows else ['-Wno-unused-function'] extensions.extend(cythonize( - [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], extra_compile_args=['-Wno-unused-function']) for m in cython_candidates], + [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], extra_compile_args=compile_args) for m in cython_candidates], exclude_failures=True)) except ImportError: warnings.warn("Cython is not installed. Not compiling core driver files as extensions (optional).") @@ -257,26 +274,6 @@ def run_setup(extensions): sys.argv = [a for a in sys.argv if a not in ("--no-murmur3", "--no-libev", "--no-cython", "--no-extensions")] -is_windows = os.name == 'nt' -if is_windows: - # libev is difficult to build, and uses select in Windows. - try: - extensions.remove(libev_ext) - except ValueError: - pass - build_extensions.error_message = """ -=============================================================================== -WARNING: could not compile %s. - -The C extensions are not required for the driver to run, but they add support -for token-aware routing with the Murmur3Partitioner. - -On Windows, make sure Visual Studio or an SDK is installed, and your environment -is configured to build for the appropriate architecture (matching your Python runtime). -This is often a matter of using vcvarsall.bat from your install directory, or running -from a command prompt in the Visual Studio Tools Start Menu. -=============================================================================== -""" platform_unsupported_msg = \ """ From 0ef3f6701d577e0f4d6a7e6753143bf485d674b6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 11:49:14 -0500 Subject: [PATCH 0244/2431] cqle: don't depend on position of keyword argument in mock call assertions Makes tests more robust, and able to run when core is cythonized Also removed autospec; autospec introspection introduced in Python 3.4 did not play well with cython method. --- .../cqlengine/query/test_batch_query.py | 8 +++---- .../cqlengine/query/test_queryset.py | 24 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/integration/cqlengine/query/test_batch_query.py b/tests/integration/cqlengine/query/test_batch_query.py index d9dc33c22d..4a3e21318c 100644 --- a/tests/integration/cqlengine/query/test_batch_query.py +++ b/tests/integration/cqlengine/query/test_batch_query.py @@ -187,13 +187,13 @@ def test_batch_execute_on_exception_skips_if_not_specified(self): self.assertEqual(0, len(obj)) def test_batch_execute_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: with BatchQuery(timeout=1) as b: BatchQueryLogModel.batch(b).create(k=2, v=2) - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=1) + self.assertEqual(mock_execute.call_args[-1]['timeout'], 1) def test_batch_execute_no_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: with BatchQuery() as b: BatchQueryLogModel.batch(b).create(k=2, v=2) - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=NOT_SET) + self.assertEqual(mock_execute.call_args[-1]['timeout'], NOT_SET) diff --git a/tests/integration/cqlengine/query/test_queryset.py b/tests/integration/cqlengine/query/test_queryset.py index 5c2b76f985..7bb101b92b 100644 --- a/tests/integration/cqlengine/query/test_queryset.py +++ b/tests/integration/cqlengine/query/test_queryset.py @@ -712,19 +712,19 @@ class PagingTest(Model): class ModelQuerySetTimeoutTestCase(BaseQuerySetUsage): def test_default_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: list(TestModel.objects()) - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=NOT_SET) + self.assertEqual(mock_execute.call_args[-1]['timeout'], NOT_SET) def test_float_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: list(TestModel.objects().timeout(0.5)) - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=0.5) + self.assertEqual(mock_execute.call_args[-1]['timeout'], 0.5) def test_none_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: list(TestModel.objects().timeout(None)) - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=None) + self.assertEqual(mock_execute.call_args[-1]['timeout'], None) class DMLQueryTimeoutTestCase(BaseQuerySetUsage): @@ -733,19 +733,19 @@ def setUp(self): super(DMLQueryTimeoutTestCase, self).setUp() def test_default_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: self.model.save() - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=NOT_SET) + self.assertEqual(mock_execute.call_args[-1]['timeout'], NOT_SET) def test_float_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: self.model.timeout(0.5).save() - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=0.5) + self.assertEqual(mock_execute.call_args[-1]['timeout'], 0.5) def test_none_timeout(self): - with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: + with mock.patch.object(Session, 'execute') as mock_execute: self.model.timeout(None).save() - mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=None) + self.assertEqual(mock_execute.call_args[-1]['timeout'], None) def test_timeout_then_batch(self): b = query.BatchQuery() From 71342f6c9db6e1592620c438bf0cdc27eefc4a22 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 11:51:40 -0500 Subject: [PATCH 0245/2431] Make tox unit tests run from alternate directory Ensures that build, installed extensions are used, instead of relying on what is inplace for the package under test. This allows us to test differing combinations of optional extensions (cython) without any assumptions about the local package state. Also: - consolidate py26 section and place unittest2 as conditional dep - introduce conditional cython dep, based on env var --- tox.ini | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tox.ini b/tox.ini index 23b2aee1f4..433e254004 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pypy,py33,py34 +envlist = py{26,27,33,34},pypy [base] deps = nose @@ -11,19 +11,15 @@ deps = nose deps = {[base]deps} sure==1.2.3 blist + {env:CYTHON_DEP:} + py26: unittest2 setenv = USE_CASS_EXTERNAL=1 -commands = {envpython} setup.py build_ext --inplace - nosetests --verbosity=2 tests/unit/ - nosetests --verbosity=2 tests/integration/cqlengine - -[testenv:py26] -deps = {[testenv]deps} - unittest2 -# test skipping is different in unittest2 for python 2.7+; let's just use it where needed +changedir = {envtmpdir} +commands = nosetests --verbosity=2 --no-path-adjustment {toxinidir}/tests/unit/ + nosetests --verbosity=2 --no-path-adjustment {toxinidir}/tests/integration/cqlengine [testenv:pypy] deps = {[base]deps} -commands = {envpython} setup.py build_ext --inplace - nosetests --verbosity=2 tests/unit/ +commands = nosetests --verbosity=2 {toxinidir}/tests/unit/ # cqlengine/test_timestamp.py uses sure, which fails in pypy presently # could remove sure usage From aa328356605cfefe54903f8b1396b01befa85937 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 11:54:36 -0500 Subject: [PATCH 0246/2431] Add travis env configs to test optional cython extensions --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 83ffff8fe1..3aa6228b92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,16 @@ language: python python: 2.7 env: - TOX_ENV=py26 CASS_VER=21 + - TOX_ENV=py26 CASS_VER=21 CYTHON_DEP=cython - TOX_ENV=py27 CASS_VER=12 - TOX_ENV=py27 CASS_VER=20 - TOX_ENV=py27 CASS_VER=21 + - TOX_ENV=py27 CASS_VER=21 CYTHON_DEP=cython - TOX_ENV=pypy CASS_VER=21 - TOX_ENV=py33 CASS_VER=21 + - TOX_ENV=py33 CASS_VER=21 CYTHON_DEP=cython - TOX_ENV=py34 CASS_VER=21 + - TOX_ENV=py34 CASS_VER=21 CYTHON_DEP=cython before_install: - sudo apt-get update -y From 376cb0c10c2a8a77eb4d7df944c18e314012aeef Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 12:44:49 -0500 Subject: [PATCH 0247/2431] print instead of warn when not building cython extensions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b5a08ab21c..54527d1a49 100644 --- a/setup.py +++ b/setup.py @@ -267,7 +267,7 @@ def run_setup(extensions): [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], extra_compile_args=compile_args) for m in cython_candidates], exclude_failures=True)) except ImportError: - warnings.warn("Cython is not installed. Not compiling core driver files as extensions (optional).") + print("Cython is not installed. Not compiling core driver files as extensions (optional).") if "--no-extensions" in sys.argv: extensions = [] From 5030c35d5312c55b238e191112ef586d53498329 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 14:12:48 -0500 Subject: [PATCH 0248/2431] release procedure: add step to update cassandra-test branch --- README-dev.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README-dev.rst b/README-dev.rst index 3d1f9b19f8..697b83f16a 100644 --- a/README-dev.rst +++ b/README-dev.rst @@ -23,6 +23,11 @@ Releasing * After a beta or rc release, this should look like ``(2, 1, '0b1', 'post')`` * Commit and push +* Update 'cassandra-test' branch to reflect new release + + * this is typically a matter of merging or rebasing onto master + * test and push updated branch to origin + * Update the JIRA versions: https://datastax-oss.atlassian.net/plugins/servlet/project-config/PYTHON/versions * Make an announcement on the mailing list From e9719ae24ef4161ada02d8f8fbf41a3ed1df5cfc Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 16:07:39 -0500 Subject: [PATCH 0249/2431] Swallow KeyErr for meta DROP updates with missing keyspace Makes control connection robust against out-of-order async events when it doesn't matter (if KS is already gone, sub-component is dropped). --- cassandra/metadata.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 8d550cee41..6e98d51336 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -210,31 +210,37 @@ def usertype_changed(self, keyspace, name, type_results): new_usertype = self._build_usertype(keyspace, type_results[0]) self.keyspaces[keyspace].user_types[name] = new_usertype else: - # the type was deleted - self.keyspaces[keyspace].user_types.pop(name, None) + try: + self.keyspaces[keyspace].user_types.pop(name, None) + except KeyError: + pass def function_changed(self, keyspace, function, function_results): if function_results: new_function = self._build_function(keyspace, function_results[0]) self.keyspaces[keyspace].functions[function.signature] = new_function else: - # the function was deleted - self.keyspaces[keyspace].functions.pop(function.signature, None) + try: + self.keyspaces[keyspace].functions.pop(function.signature, None) + except KeyError: + pass def aggregate_changed(self, keyspace, aggregate, aggregate_results): if aggregate_results: new_aggregate = self._build_aggregate(keyspace, aggregate_results[0]) self.keyspaces[keyspace].aggregates[aggregate.signature] = new_aggregate else: - # the aggregate was deleted - self.keyspaces[keyspace].aggregates.pop(aggregate.signature, None) + try: + self.keyspaces[keyspace].aggregates.pop(aggregate.signature, None) + except KeyError: + pass def table_changed(self, keyspace, table, cf_results, col_results, triggers_result): try: keyspace_meta = self.keyspaces[keyspace] except KeyError: # we're trying to update a table in a keyspace we don't know about - log.error("Tried to update schema for table '%s' in unknown keyspace '%s'", + log.warn("Tried to update schema for table '%s' in unknown keyspace '%s'", table, keyspace) return From 5777c204698543e1c18870173a2b3417f668324e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 9 Jul 2015 16:53:51 -0500 Subject: [PATCH 0250/2431] Attempt to process correlated events the order in which the arrive Events are mostly processed in order. "mostly" because there is still a race by way of the cluster thread pool executor. Meta class should be robust against this. --- cassandra/cluster.py | 22 +++++++++++++++++++--- cassandra/metadata.py | 19 +++++++------------ tests/unit/test_control_connection.py | 4 ++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 45784c4623..fb697a182a 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2065,6 +2065,8 @@ def __init__(self, cluster, timeout, self._reconnection_handler = None self._reconnection_lock = RLock() + self._event_schedule_times = {} + def connect(self): if self._is_shutdown: return @@ -2501,12 +2503,26 @@ def _update_location_info(self, host, datacenter, rack): self._cluster.load_balancing_policy.on_up(host) return True + def _delay_for_event_type(self, event_type, delay_window): + # this serves to order processing correlated events (received within the window) + # the window and randomization still have the desired effect of skew across client instances + next_time = self._event_schedule_times.get(event_type, 0) + now = self._time.time() + if now <= next_time: + this_time = next_time + 0.01 + delay = this_time - now + else: + delay = random() * delay_window + this_time = now + delay + self._event_schedule_times[event_type] = this_time + return delay + def _handle_topology_change(self, event): change_type = event["change_type"] addr, port = event["address"] if change_type == "NEW_NODE" or change_type == "MOVED_NODE": if self._topology_event_refresh_window >= 0: - delay = random() * self._topology_event_refresh_window + delay = self._delay_for_event_type('topology_change', self._topology_event_refresh_window) self._cluster.scheduler.schedule_unique(delay, self.refresh_node_list_and_token_map) elif change_type == "REMOVED_NODE": host = self._cluster.metadata.get_host(addr) @@ -2517,7 +2533,7 @@ def _handle_status_change(self, event): addr, port = event["address"] host = self._cluster.metadata.get_host(addr) if change_type == "UP": - delay = 1 + random() * 0.5 # randomness to avoid thundering herd problem on events + delay = 1 + self._delay_for_event_type('status_change', 0.5) # randomness to avoid thundering herd problem on events if host is None: # this is the first time we've seen the node self._cluster.scheduler.schedule_unique(delay, self.refresh_node_list_and_token_map) @@ -2540,7 +2556,7 @@ def _handle_schema_change(self, event): usertype = event.get('type') function = event.get('function') aggregate = event.get('aggregate') - delay = random() * self._schema_event_refresh_window + delay = self._delay_for_event_type('schema_change', self._schema_event_refresh_window) self._cluster.scheduler.schedule_unique(delay, self.refresh_schema, keyspace, table, usertype, function, aggregate) def wait_for_schema_agreement(self, connection=None, preloaded_results=None, wait_time=None): diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 6e98d51336..f977e0ffae 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -236,21 +236,16 @@ def aggregate_changed(self, keyspace, aggregate, aggregate_results): pass def table_changed(self, keyspace, table, cf_results, col_results, triggers_result): - try: - keyspace_meta = self.keyspaces[keyspace] - except KeyError: - # we're trying to update a table in a keyspace we don't know about - log.warn("Tried to update schema for table '%s' in unknown keyspace '%s'", - table, keyspace) - return - - if not cf_results: - # the table was removed - keyspace_meta._drop_table_metadata(table) - else: + if cf_results: assert len(cf_results) == 1 + keyspace_meta = self.keyspaces[keyspace] table_meta = self._build_table_metadata(keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) keyspace_meta._add_table_metadata(table_meta) + else: + try: + self.keyspaces[keyspace]._drop_table_metadata(table) + except KeyError: + pass def _keyspace_added(self, ksname): if self.token_map: diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 9c93f5afe4..1429531c2d 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -410,12 +410,12 @@ def test_handle_schema_change(self): } self.cluster.scheduler.reset_mock() self.control_connection._handle_schema_change(event) - self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', 'table1', None, None, None) + self.cluster.scheduler.schedule_unique.assert_called_once_with(ANY, self.control_connection.refresh_schema, 'ks1', 'table1', None, None, None) self.cluster.scheduler.reset_mock() event['table'] = None self.control_connection._handle_schema_change(event) - self.cluster.scheduler.schedule_unique.assert_called_once_with(0.0, self.control_connection.refresh_schema, 'ks1', None, None, None, None) + self.cluster.scheduler.schedule_unique.assert_called_once_with(ANY, self.control_connection.refresh_schema, 'ks1', None, None, None, None) def test_refresh_disabled(self): cluster = MockCluster() From fafb29d622d3baa79aef0cee96a5c8e03e27d3e8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Jul 2015 09:52:48 -0500 Subject: [PATCH 0251/2431] Limit mock dependency to version 1.0.1 1.1.0 dropped Python 2.6 support, and at least one call assert function currently used by tests. --- setup.py | 2 +- test-requirements.txt | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e85d332d37..ea6f4f0514 100644 --- a/setup.py +++ b/setup.py @@ -216,7 +216,7 @@ def run_setup(extensions): keywords='cassandra,cql,orm', include_package_data=True, install_requires=dependencies, - tests_require=['nose', 'mock', 'PyYAML', 'pytz', 'sure'], + tests_require=['nose', 'mock<=1.0.1', 'PyYAML', 'pytz', 'sure'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/test-requirements.txt b/test-requirements.txt index a90f1ad5c5..cc7b3c2b2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ blist scales nose -mock +mock<=1.0.1 ccm>=2.0 unittest2 PyYAML diff --git a/tox.ini b/tox.ini index 23b2aee1f4..b6b0b519dc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py26,py27,pypy,py33,py34 [base] deps = nose - mock + mock<=1.0.1 PyYAML six From 870d76946b90de2fdec887ac238ecf71f1145c20 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Jul 2015 09:52:48 -0500 Subject: [PATCH 0252/2431] Limit mock dependency to version 1.0.1 1.1.0 dropped Python 2.6 support, and at least one call assert function currently used by tests. --- setup.py | 2 +- test-requirements.txt | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e85d332d37..ea6f4f0514 100644 --- a/setup.py +++ b/setup.py @@ -216,7 +216,7 @@ def run_setup(extensions): keywords='cassandra,cql,orm', include_package_data=True, install_requires=dependencies, - tests_require=['nose', 'mock', 'PyYAML', 'pytz', 'sure'], + tests_require=['nose', 'mock<=1.0.1', 'PyYAML', 'pytz', 'sure'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/test-requirements.txt b/test-requirements.txt index a90f1ad5c5..cc7b3c2b2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ blist scales nose -mock +mock<=1.0.1 ccm>=2.0 unittest2 PyYAML diff --git a/tox.ini b/tox.ini index 23b2aee1f4..b6b0b519dc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py26,py27,pypy,py33,py34 [base] deps = nose - mock + mock<=1.0.1 PyYAML six From 1f0c6c312bce36d0892e18725baace3a0ae02a28 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Jul 2015 09:52:48 -0500 Subject: [PATCH 0253/2431] Limit mock dependency to version 1.0.1 1.1.0 dropped Python 2.6 support, and at least one call assert function currently used by tests. --- setup.py | 2 +- test-requirements.txt | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e85d332d37..ea6f4f0514 100644 --- a/setup.py +++ b/setup.py @@ -216,7 +216,7 @@ def run_setup(extensions): keywords='cassandra,cql,orm', include_package_data=True, install_requires=dependencies, - tests_require=['nose', 'mock', 'PyYAML', 'pytz', 'sure'], + tests_require=['nose', 'mock<=1.0.1', 'PyYAML', 'pytz', 'sure'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/test-requirements.txt b/test-requirements.txt index a90f1ad5c5..cc7b3c2b2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ blist scales nose -mock +mock<=1.0.1 ccm>=2.0 unittest2 PyYAML diff --git a/tox.ini b/tox.ini index 23b2aee1f4..b6b0b519dc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py26,py27,pypy,py33,py34 [base] deps = nose - mock + mock<=1.0.1 PyYAML six From 6be945b6e9061ddf416970a398a7c2ae9355dced Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Jul 2015 17:06:33 -0500 Subject: [PATCH 0254/2431] Initial version of generator-based concurrent executor not fully tested --- cassandra/concurrent.py | 86 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index b96c921bd2..81c6938178 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -24,7 +24,6 @@ log = logging.getLogger(__name__) - def execute_concurrent(session, statements_and_parameters, concurrency=100, raise_on_first_error=True): """ Executes a sequence of (statement, parameters) tuples concurrently. Each @@ -74,11 +73,6 @@ def execute_concurrent(session, statements_and_parameters, concurrency=100, rais if not statements_and_parameters: return [] - # TODO handle iterators and generators naturally without converting the - # whole thing to a list. This would require not building a result - # list of Nones up front (we don't know how many results there will be), - # so a dict keyed by index should be used instead. The tricky part is - # knowing when you're the final statement to finish. statements_and_parameters = list(statements_and_parameters) event = Event() @@ -101,6 +95,86 @@ def execute_concurrent(session, statements_and_parameters, concurrency=100, rais else: return results +from threading import Lock, Condition +from heapq import heappush, heappop + +class ConcurrentExecutor(object): + + def __init__(self, session, statements_and_params): + self.session = session + self._enum_statements = enumerate(iter(statements_and_params)) + self._lock = Lock() + self._condition = Condition(self._lock) + self._fail_fast = True + self._results_queue = [] + self._exec_count = 0 + + def execute(self, concurrency, fail_fast): + self._fail_fast = fail_fast + self._results_queue = [] + self._exec_count = 0 + with self._lock: + for n in xrange(concurrency): + if not self._execute_next(): + break + return self._results() + + def _execute_next(self): + # lock must be held + try: + (idx, (statement, params)) = next(self._enum_statements) + self._exec_count += 1 + self._execute(idx, statement, params) + return True + except StopIteration: + pass + + def _execute(self, idx, statement, params): + try: + future = self.session.execute_async(statement, params, timeout=None) + args = (idx,) + future.add_callbacks( + callback=self._on_success, callback_args=args, + errback=self._on_error, errback_args=args) + except Exception as exc: + e = sys.exc_info() if six.PY2 else exc + self._on_error(e, idx) + + def _on_success(self, result, idx): + self._put_result(idx, True, result) + + def _on_error(self, result, idx): + self._put_result(idx, False, result) + + def _put_result(self, idx, success, result): + with self._condition: + heappush(self._results_queue, (idx, (success, result))) + self._execute_next() + self._condition.notify() + + def _results(self): + # todo: done condition + current = 0 + with self._condition: + while current < self._exec_count: + while not self._results_queue or self._results_queue[0][0] != current: + self._condition.wait() + while self._results_queue and self._results_queue[0][0] == current: + _, res = heappop(self._results_queue) + if self._fail_fast and not res[0]: + self._raise(res[1]) + yield res + current += 1 + + @staticmethod + def _raise(exc): + if six.PY2 and isinstance(exc, tuple): + (exc_type, value, traceback) = exc + six.reraise(exc_type, value, traceback) + else: + raise exc + + def execute_concurrent_with_args(session, statement, parameters, *args, **kwargs): """ From 6eebdebd64de676188f8c158f6b620b8dba9cbad Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 13 Jul 2015 17:40:11 -0500 Subject: [PATCH 0255/2431] Another iteration on updated schema This version parses the new tables schema. API is a little messy -- I would like to revisit that. PYTHON-276 --- cassandra/metadata.py | 176 +++++++++++++++----- tests/integration/standard/test_metadata.py | 6 +- 2 files changed, 134 insertions(+), 48 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 6b177d7049..8bca0beed1 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -960,30 +960,6 @@ def primary_key(self): table. """ - recognized_options = ( - "comment", - "read_repair_chance", - "dclocal_read_repair_chance", # kept to be safe, but see _build_table_options() - "local_read_repair_chance", - "replicate_on_write", - "gc_grace_seconds", - "bloom_filter_fp_chance", - "caching", - "compaction_strategy_class", - "compaction_strategy_options", - "min_compaction_threshold", - "max_compaction_threshold", - "compression_parameters", - "min_index_interval", - "max_index_interval", - "index_interval", - "speculative_retry", - "rows_per_partition_to_cache", - "memtable_flush_period_in_ms", - "populate_io_cache_on_flush", - "compression", - "default_time_to_live") - compaction_options = { "min_compaction_threshold": "min_threshold", "max_compaction_threshold": "max_threshold", @@ -999,16 +975,19 @@ def is_cql_compatible(self): """ A boolean indicating if this table can be represented as CQL in export """ - # no such thing as DCT in CQL - incompatible = issubclass(self.comparator, types.DynamicCompositeType) + comparator = getattr(self, 'comparator', None) + if comparator: + # no such thing as DCT in CQL + incompatible = issubclass(self.comparator, types.DynamicCompositeType) - # no compact storage with more than one column beyond PK if there - # are clustering columns - incompatible |= (self.is_compact_storage and - len(self.columns) > len(self.primary_key) + 1 and - len(self.clustering_key) >= 1) + # no compact storage with more than one column beyond PK if there + # are clustering columns + incompatible |= (self.is_compact_storage and + len(self.columns) > len(self.primary_key) + 1 and + len(self.clustering_key) >= 1) - return not incompatible + return not incompatible + return True def __init__(self, keyspace_name, name, partition_key=None, clustering_key=None, columns=None, triggers=None, options=None): self.keyspace_name = keyspace_name @@ -1101,18 +1080,10 @@ def as_cql_query(self, formatted=False): if self.clustering_key: cluster_str = "CLUSTERING ORDER BY " - clustering_names = protect_names([c.name for c in self.clustering_key]) - - if self.is_compact_storage and \ - not issubclass(self.comparator, types.CompositeType): - subtypes = [self.comparator] - else: - subtypes = self.comparator.subtypes - inner = [] - for colname, coltype in zip(clustering_names, subtypes): - ordering = "DESC" if issubclass(coltype, types.ReversedType) else "ASC" - inner.append("%s %s" % (colname, ordering)) + for col in self.clustering_key: + ordering = "DESC" if issubclass(col.data_type, types.ReversedType) else "ASC" + inner.append("%s %s" % (protect_name(col.name), ordering)) cluster_str += "(%s)" % ", ".join(inner) option_strings.append(cluster_str) @@ -1545,6 +1516,30 @@ class SchemaParserV22(_SchemaParser): _table_name_col = 'columnfamily_name' + recognized_table_options = ( + "comment", + "read_repair_chance", + "dclocal_read_repair_chance", # kept to be safe, but see _build_table_options() + "local_read_repair_chance", + "replicate_on_write", + "gc_grace_seconds", + "bloom_filter_fp_chance", + "caching", + "compaction_strategy_class", + "compaction_strategy_options", + "min_compaction_threshold", + "max_compaction_threshold", + "compression_parameters", + "min_index_interval", + "max_index_interval", + "index_interval", + "speculative_retry", + "rows_per_partition_to_cache", + "memtable_flush_period_in_ms", + "populate_io_cache_on_flush", + "compression", + "default_time_to_live") + def __init__(self, connection, timeout): super(SchemaParserV22, self).__init__(connection, timeout) self.keyspaces_result = [] @@ -1809,10 +1804,9 @@ def _build_table_metadata(self, row, col_rows, trigger_rows): return table_meta - @staticmethod - def _build_table_options(row): + def _build_table_options(self, row): """ Setup the mostly-non-schema table options, like caching settings """ - options = dict((o, row.get(o)) for o in TableMetadata.recognized_options if o in row) + options = dict((o, row.get(o)) for o in self.recognized_table_options if o in row) # the option name when creating tables is "dclocal_read_repair_chance", # but the column name in system.schema_columnfamilies is @@ -1960,6 +1954,21 @@ class SchemaParserV3(SchemaParserV22): _table_name_col = 'table_name' + recognized_table_options = ( + 'bloom_filter_fp_chance', + 'caching', + 'comment', + 'compaction', + 'compression', + 'dclocal_read_repair_chance', + 'default_time_to_live', + 'gc_grace_seconds', + 'max_index_interval', + 'memtable_flush_period_in_ms', + 'min_index_interval', + 'read_repair_chance', + 'speculative_retry') + @staticmethod def _build_keyspace_metadata(row): name = row["keyspace_name"] @@ -1968,6 +1977,81 @@ def _build_keyspace_metadata(row): strategy_class = strategy_options.pop("class") return KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) + def _build_table_metadata(self, row, col_rows, trigger_rows): + keyspace_name = row["keyspace_name"] + table_name = row[self._table_name_col] + cf_col_rows = col_rows.get(table_name, []) + + if not cf_col_rows: # CASSANDRA-8487 + log.warning("Building table metadata with no column meta for %s.%s", + keyspace_name, table_name) + + table_meta = TableMetadataV3(keyspace_name, table_name) + + for col_row in cf_col_rows: + column_meta = self._build_column_metadata(table_meta, col_row) + table_meta.columns[column_meta.name] = column_meta + + # partition key + partition_rows = [r for r in cf_col_rows + if r.get('type', None) == "partition_key"] + if len(partition_rows) > 1: + partition_rows = sorted(partition_rows, key=lambda row: row.get('component_index')) + for r in partition_rows: + table_meta.partition_key.append(table_meta.columns[r.get('column_name')]) + + # clustering key + clustering_rows = [r for r in cf_col_rows + if r.get('type', None) == "clustering"] + if len(clustering_rows) > 1: + clustering_rows = sorted(clustering_rows, key=lambda row: row.get('component_index')) + for r in clustering_rows: + table_meta.clustering_key.append(table_meta.columns[r.get('column_name')]) + + if trigger_rows: + for trigger_row in trigger_rows[table_name]: + trigger_meta = self._build_trigger_metadata(table_meta, trigger_row) + table_meta.triggers[trigger_meta.name] = trigger_meta + + table_meta.options = self._build_table_options(row) + flags = row.get('flags', set()) + if flags: + table_meta.is_compact_storage = 'dense' in flags or 'super' in flags or 'compound' not in flags + return table_meta + + def _build_table_options(self, row): + """ Setup the mostly-non-schema table options, like caching settings """ + return dict((o, row.get(o)) for o in self.recognized_table_options if o in row) + +class TableMetadataV3(TableMetadata): + """ + For now, until I figure out what to do with this API + """ + compaction_options = {} + + _option_maps = ['caching'] + option_maps = ['compaction', 'compression', 'caching'] + + @property + def is_cql_compatible(self): + return True + + def _make_option_strings(self): + ret = [] + options_copy = dict(self.options.items()) + + for option in self.option_maps: + value = options_copy.pop(option, {}) + params = ("'%s': '%s'" % (k, v) for k, v in value.items()) + ret.append("%s = {%s}" % (option, ', '.join(params))) + + for name, value in options_copy.items(): + if value is not None: + if name == "comment": + value = value or "" + ret.append("%s = %s" % (name, protect_value(value))) + + return list(sorted(ret)) def get_schema_parser(connection, timeout): server_version = connection.server_version diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index 582977a95e..3152fdb311 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -28,7 +28,7 @@ from cassandra.cqltypes import DoubleType, Int32Type, ListType, UTF8Type, MapType from cassandra.encoder import Encoder from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, IndexMetadata, - Token, MD5Token, TokenMap, murmur3, Function, Aggregate) + Token, MD5Token, TokenMap, murmur3, Function, Aggregate, get_schema_parser) from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host @@ -132,8 +132,10 @@ def test_basic_table_meta_properties(self): self.assertEqual([], tablemeta.clustering_key) self.assertEqual([u'a', u'b', u'c'], sorted(tablemeta.columns.keys())) + parser = get_schema_parser(self.cluster.control_connection._connection, 1) + for option in tablemeta.options: - self.assertIn(option, TableMetadata.recognized_options) + self.assertIn(option, parser.recognized_table_options) self.check_create_statement(tablemeta, create_statement) From dc24bf1d0ea23c5a6713f3c30988c09446a8fa01 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 14 Jul 2015 11:26:10 -0500 Subject: [PATCH 0256/2431] schema meta: make sure columns added in key order also modify table build based on compact static condition --- cassandra/metadata.py | 45 ++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 8bca0beed1..014390bdea 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1988,9 +1988,15 @@ def _build_table_metadata(self, row, col_rows, trigger_rows): table_meta = TableMetadataV3(keyspace_name, table_name) - for col_row in cf_col_rows: - column_meta = self._build_column_metadata(table_meta, col_row) - table_meta.columns[column_meta.name] = column_meta + table_meta.options = self._build_table_options(row) + flags = row.get('flags', set()) + if flags: + compact_static = False + table_meta.is_compact_storage = 'dense' in flags or 'super' in flags or 'compound' not in flags + else: + # example: create table t (a int, b int, c int, primary key((a,b))) with compact storage + compact_static = True + table_meta.is_compact_storage = True # partition key partition_rows = [r for r in cf_col_rows @@ -1998,25 +2004,38 @@ def _build_table_metadata(self, row, col_rows, trigger_rows): if len(partition_rows) > 1: partition_rows = sorted(partition_rows, key=lambda row: row.get('component_index')) for r in partition_rows: + # we have to add meta here (and not in the later loop) because TableMetadata.columns is an + # OrderedDict, and it assumes keys are inserted first, in order, when exporting CQL + column_meta = self._build_column_metadata(table_meta, r) + table_meta.columns[column_meta.name] = column_meta table_meta.partition_key.append(table_meta.columns[r.get('column_name')]) # clustering key - clustering_rows = [r for r in cf_col_rows - if r.get('type', None) == "clustering"] - if len(clustering_rows) > 1: - clustering_rows = sorted(clustering_rows, key=lambda row: row.get('component_index')) - for r in clustering_rows: - table_meta.clustering_key.append(table_meta.columns[r.get('column_name')]) + if not compact_static: + clustering_rows = [r for r in cf_col_rows + if r.get('type', None) == "clustering"] + if len(clustering_rows) > 1: + clustering_rows = sorted(clustering_rows, key=lambda row: row.get('component_index')) + for r in clustering_rows: + column_meta = self._build_column_metadata(table_meta, r) + table_meta.columns[column_meta.name] = column_meta + table_meta.clustering_key.append(table_meta.columns[r.get('column_name')]) + + for col_row in (r for r in cf_col_rows + if r.get('type', None) not in ('parition_key', 'clustering_key')): + column_meta = self._build_column_metadata(table_meta, col_row) + if not compact_static or column_meta.is_static: + # for compact static tables, we omit the clustering key and value, and only add the logical columns. + # They are marked not static so that it generates appropriate CQL + if compact_static: + column_meta.is_static = False + table_meta.columns[column_meta.name] = column_meta if trigger_rows: for trigger_row in trigger_rows[table_name]: trigger_meta = self._build_trigger_metadata(table_meta, trigger_row) table_meta.triggers[trigger_meta.name] = trigger_meta - table_meta.options = self._build_table_options(row) - flags = row.get('flags', set()) - if flags: - table_meta.is_compact_storage = 'dense' in flags or 'super' in flags or 'compound' not in flags return table_meta def _build_table_options(self, row): From 46222fa5b01d98dd7a3413a545f6aa2a3d65df5b Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 14 Jul 2015 16:13:50 -0500 Subject: [PATCH 0257/2431] Allow generator for inputs and outputs of execute_concurrent PYTHON-123 --- cassandra/concurrent.py | 236 ++++++++++++++++------------------------ 1 file changed, 91 insertions(+), 145 deletions(-) diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index 81c6938178..aba3045605 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -12,33 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. + +from heapq import heappush, heappop +from itertools import cycle import six +from six.moves import xrange, zip +from threading import Condition import sys -from itertools import count, cycle -import logging -from six.moves import xrange -from threading import Event - from cassandra.cluster import PagedResult +import logging log = logging.getLogger(__name__) -def execute_concurrent(session, statements_and_parameters, concurrency=100, raise_on_first_error=True): +def execute_concurrent(session, statements_and_parameters, concurrency=100, raise_on_first_error=True, results_generator=False): """ Executes a sequence of (statement, parameters) tuples concurrently. Each ``parameters`` item must be a sequence or :const:`None`. - A sequence of ``(success, result_or_exc)`` tuples is returned in the same - order that the statements were passed in. If ``success`` is :const:`False`, - there was an error executing the statement, and ``result_or_exc`` will be - an :class:`Exception`. If ``success`` is :const:`True`, ``result_or_exc`` - will be the query result. - - If `raise_on_first_error` is left as :const:`True`, execution will stop - after the first failed statement and the corresponding exception will be - raised. - The `concurrency` parameter controls how many statements will be executed concurrently. When :attr:`.Cluster.protocol_version` is set to 1 or 2, it is recommended that this be kept below 100 times the number of @@ -48,6 +39,26 @@ def execute_concurrent(session, statements_and_parameters, concurrency=100, rais substantially impacting throughput. If :attr:`~.Cluster.protocol_version` is 3 or higher, you can safely experiment with higher levels of concurrency. + If `raise_on_first_error` is left as :const:`True`, execution will stop + after the first failed statement and the corresponding exception will be + raised. + + `results_generator` controls how the results are returned. + + If :const:`False`, the results are returned only after all requests have completed. + + If :const:`True`, a generator expression is returned. Using a generator results in + a constrained memory footprint when the results set will be large -- results are yielded + as they return instead of materializing the entire list at once. The trade for lower memory + footprint is marginal CPU overhead (more thread coordination and sorting out-of-order results + on-the-fly). + + A sequence of ``(success, result_or_exc)`` tuples is returned in the same + order that the statements were passed in. If ``success`` is :const:`False`, + there was an error executing the statement, and ``result_or_exc`` will be + an :class:`Exception`. If ``success`` is :const:`True`, ``result_or_exc`` + will be the query result. + Example usage:: select_statement = session.prepare("SELECT * FROM users WHERE id=?") @@ -73,47 +84,27 @@ def execute_concurrent(session, statements_and_parameters, concurrency=100, rais if not statements_and_parameters: return [] - statements_and_parameters = list(statements_and_parameters) + executor = ConcurrentExecutorGenResults(session, statements_and_parameters) if results_generator else ConcurrentExecutorListResults(session, statements_and_parameters) + return executor.execute(concurrency, raise_on_first_error) - event = Event() - first_error = [] if raise_on_first_error else None - to_execute = len(statements_and_parameters) - results = [None] * to_execute - num_finished = count(1) - statements = enumerate(iter(statements_and_parameters)) - for i in xrange(min(concurrency, len(statements_and_parameters))): - _execute_next(_sentinel, i, event, session, statements, results, None, num_finished, to_execute, first_error) - event.wait() - if first_error: - exc = first_error[0] - if six.PY2 and isinstance(exc, tuple): - (exc_type, value, traceback) = exc - six.reraise(exc_type, value, traceback) - else: - raise exc - else: - return results - -from threading import Lock, Condition -from heapq import heappush, heappop - -class ConcurrentExecutor(object): +class _ConcurrentExecutor(object): def __init__(self, session, statements_and_params): self.session = session self._enum_statements = enumerate(iter(statements_and_params)) - self._lock = Lock() - self._condition = Condition(self._lock) - self._fail_fast = True + self._condition = Condition() + self._fail_fast = False self._results_queue = [] + self._current = 0 self._exec_count = 0 def execute(self, concurrency, fail_fast): self._fail_fast = fail_fast self._results_queue = [] + self._current = 0 self._exec_count = 0 - with self._lock: + with self._condition: for n in xrange(concurrency): if not self._execute_next(): break @@ -132,47 +123,81 @@ def _execute_next(self): def _execute(self, idx, statement, params): try: future = self.session.execute_async(statement, params, timeout=None) - args = (idx,) + args = (future, idx) future.add_callbacks( callback=self._on_success, callback_args=args, errback=self._on_error, errback_args=args) except Exception as exc: - e = sys.exc_info() if six.PY2 else exc - self._on_error(e, idx) + # exc_info with fail_fast to preserve stack trace info when raising on the client thread + # (matches previous behavior -- not sure why we wouldn't want stack trace in the other case) + e = sys.exc_info() if self._fail_fast and six.PY2 else exc + self._put_result(e, idx, False) - def _on_success(self, result, idx): - self._put_result(idx, True, result) + def _on_success(self, result, future, idx): + if future.has_more_pages: + result = PagedResult(future, result) + future.clear_callbacks() + self._put_result(result, idx, True) + + def _on_error(self, result, future, idx): + self._put_result(result, idx, False) - def _on_error(self, result, idx): - self._put_result(idx, False, result) + @staticmethod + def _raise(exc): + if six.PY2 and isinstance(exc, tuple): + (exc_type, value, traceback) = exc + six.reraise(exc_type, value, traceback) + else: + raise exc - def _put_result(self, idx, success, result): + +class ConcurrentExecutorGenResults(_ConcurrentExecutor): + + def _put_result(self, result, idx, success): with self._condition: heappush(self._results_queue, (idx, (success, result))) self._execute_next() self._condition.notify() def _results(self): - # todo: done condition - current = 0 with self._condition: - while current < self._exec_count: - while not self._results_queue or self._results_queue[0][0] != current: + while self._current < self._exec_count: + while not self._results_queue or self._results_queue[0][0] != self._current: self._condition.wait() - while self._results_queue and self._results_queue[0][0] == current: + while self._results_queue and self._results_queue[0][0] == self._current: _, res = heappop(self._results_queue) if self._fail_fast and not res[0]: self._raise(res[1]) yield res - current += 1 + self._current += 1 - @staticmethod - def _raise(exc): - if six.PY2 and isinstance(exc, tuple): - (exc_type, value, traceback) = exc - six.reraise(exc_type, value, traceback) - else: - raise exc + +class ConcurrentExecutorListResults(_ConcurrentExecutor): + + _exception = None + + def execute(self, concurrency, fail_fast): + self._exception = None + return super(ConcurrentExecutorListResults, self).execute(concurrency, fail_fast) + + def _put_result(self, result, idx, success): + self._results_queue.append((idx, (success, result))) + with self._condition: + self._current += 1 + if not success and self._fail_fast: + if not self._exception: + self._exception = result + self._condition.notify() + elif not self._execute_next() and self._current == self._exec_count: + self._condition.notify() + + def _results(self): + with self._condition: + while self._current < self._exec_count: + self._condition.wait() + if self._exception and self._fail_fast: + self._raise(self._exception) + return [r[1] for r in sorted(self._results_queue)] @@ -188,83 +213,4 @@ def execute_concurrent_with_args(session, statement, parameters, *args, **kwargs parameters = [(x,) for x in range(1000)] execute_concurrent_with_args(session, statement, parameters, concurrency=50) """ - return execute_concurrent(session, list(zip(cycle((statement,)), parameters)), *args, **kwargs) - - -_sentinel = object() - - -def _handle_error(error, result_index, event, session, statements, results, - future, num_finished, to_execute, first_error): - if first_error is not None: - first_error.append(error) - event.set() - return - else: - results[result_index] = (False, error) - if next(num_finished) >= to_execute: - event.set() - return - - try: - (next_index, (statement, params)) = next(statements) - except StopIteration: - return - - try: - future = session.execute_async(statement, params, timeout=None) - args = (next_index, event, session, statements, results, future, num_finished, to_execute, first_error) - future.add_callbacks( - callback=_execute_next, callback_args=args, - errback=_handle_error, errback_args=args) - except Exception as exc: - if first_error is not None: - if six.PY2: - first_error.append(sys.exc_info()) - else: - first_error.append(exc) - event.set() - return - else: - results[next_index] = (False, exc) - if next(num_finished) >= to_execute: - event.set() - return - - -def _execute_next(result, result_index, event, session, statements, results, - future, num_finished, to_execute, first_error): - if result is not _sentinel: - if future.has_more_pages: - result = PagedResult(future, result) - future.clear_callbacks() - results[result_index] = (True, result) - finished = next(num_finished) - if finished >= to_execute: - event.set() - return - - try: - (next_index, (statement, params)) = next(statements) - except StopIteration: - return - - try: - future = session.execute_async(statement, params, timeout=None) - args = (next_index, event, session, statements, results, future, num_finished, to_execute, first_error) - future.add_callbacks( - callback=_execute_next, callback_args=args, - errback=_handle_error, errback_args=args) - except Exception as exc: - if first_error is not None: - if six.PY2: - first_error.append(sys.exc_info()) - else: - first_error.append(exc) - event.set() - return - else: - results[next_index] = (False, exc) - if next(num_finished) >= to_execute: - event.set() - return + return execute_concurrent(session, zip(cycle((statement,)), parameters), *args, **kwargs) From f7c0bd312df6ac105173969c8b57ee4b8bd9c664 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Thu, 25 Jun 2015 10:59:57 -0700 Subject: [PATCH 0258/2431] Enable C extensions on pypy and add pypy3 support Conflicts: tox.ini --- cassandra/murmur3.c | 10 +++++++++- setup.py | 4 ++-- tox.ini | 11 +++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/cassandra/murmur3.c b/cassandra/murmur3.c index 657c01f69d..dc98b4919d 100644 --- a/cassandra/murmur3.c +++ b/cassandra/murmur3.c @@ -20,6 +20,13 @@ typedef int Py_ssize_t; #define PY_SSIZE_T_MIN INT_MIN #endif +#ifdef PYPY_VERSION +#define COMPILING_IN_PYPY 1 +#define COMPILING_IN_CPYTHON 0 +#else +#define COMPILING_IN_PYPY 0 +#define COMPILING_IN_CPYTHON 1 +#endif //----------------------------------------------------------------------------- // Platform-specific functions and macros @@ -179,7 +186,8 @@ struct module_state { PyObject *error; }; -#if PY_MAJOR_VERSION >= 3 +// pypy3 doesn't have GetState yet. +#if COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) diff --git a/setup.py b/setup.py index 54527d1a49..bf12426231 100644 --- a/setup.py +++ b/setup.py @@ -235,6 +235,7 @@ def run_setup(extensions): **kw) is_windows = os.name == 'nt' + if is_windows: build_extensions.error_message = """ =============================================================================== @@ -291,8 +292,7 @@ def run_setup(extensions): if extensions: if (sys.platform.startswith("java") - or sys.platform == "cli" - or "PyPy" in sys.version): + or sys.platform == "cli"): sys.stderr.write(platform_unsupported_msg) extensions = () elif sys.byteorder == "big": diff --git a/tox.ini b/tox.ini index 433e254004..b1f99087ad 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{26,27,33,34},pypy +envlist = py{26,27,33,34},pypy,pypy3 [base] deps = nose @@ -20,6 +20,13 @@ commands = nosetests --verbosity=2 --no-path-adjustment {toxinidir}/tests/unit/ [testenv:pypy] deps = {[base]deps} -commands = nosetests --verbosity=2 {toxinidir}/tests/unit/ +commands = {envpython} setup.py build_ext --inplace + nosetests --verbosity=2 tests/unit/ + +[testenv:pypy3] +deps = {[base]deps} +commands = {envpython} setup.py build_ext --inplace + nosetests --verbosity=2 tests/unit/ + # cqlengine/test_timestamp.py uses sure, which fails in pypy presently # could remove sure usage From e08fefc9822d7d8594262e2735fa79e80d6b59f7 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 15 Jul 2015 14:48:43 -0500 Subject: [PATCH 0259/2431] Make tests.unit.io.test_twistedreactor work in Python3 --- tests/unit/io/test_twistedreactor.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/io/test_twistedreactor.py b/tests/unit/io/test_twistedreactor.py index d2142b09ca..9063460931 100644 --- a/tests/unit/io/test_twistedreactor.py +++ b/tests/unit/io/test_twistedreactor.py @@ -153,17 +153,17 @@ def test_handle_read__incomplete(self): Verify that handle_read() processes incomplete messages properly. """ self.obj_ut.process_msg = Mock() - self.assertEqual(self.obj_ut._iobuf.getvalue(), '') # buf starts empty + self.assertEqual(self.obj_ut._iobuf.getvalue(), b'') # buf starts empty # incomplete header - self.obj_ut._iobuf.write('\x84\x00\x00\x00\x00') + self.obj_ut._iobuf.write(b'\x84\x00\x00\x00\x00') self.obj_ut.handle_read() - self.assertEqual(self.obj_ut._iobuf.getvalue(), '\x84\x00\x00\x00\x00') + self.assertEqual(self.obj_ut._iobuf.getvalue(), b'\x84\x00\x00\x00\x00') # full header, but incomplete body - self.obj_ut._iobuf.write('\x00\x00\x00\x15') + self.obj_ut._iobuf.write(b'\x00\x00\x00\x15') self.obj_ut.handle_read() self.assertEqual(self.obj_ut._iobuf.getvalue(), - '\x84\x00\x00\x00\x00\x00\x00\x00\x15') + b'\x84\x00\x00\x00\x00\x00\x00\x00\x15') self.assertEqual(self.obj_ut._current_frame.end_pos, 30) # verify we never attempted to process the incomplete message @@ -174,14 +174,14 @@ def test_handle_read__fullmessage(self): Verify that handle_read() processes complete messages properly. """ self.obj_ut.process_msg = Mock() - self.assertEqual(self.obj_ut._iobuf.getvalue(), '') # buf starts empty + self.assertEqual(self.obj_ut._iobuf.getvalue(), b'') # buf starts empty # write a complete message, plus 'NEXT' (to simulate next message) # assumes protocol v3+ as default Connection.protocol_version - body = 'this is the drum roll' - extra = 'NEXT' + body = b'this is the drum roll' + extra = b'NEXT' self.obj_ut._iobuf.write( - '\x84\x01\x00\x02\x03\x00\x00\x00\x15' + body + extra) + b'\x84\x01\x00\x02\x03\x00\x00\x00\x15' + body + extra) self.obj_ut.handle_read() self.assertEqual(self.obj_ut._iobuf.getvalue(), extra) self.obj_ut.process_msg.assert_called_with( From cf7e6683d3c036b70e17de9fa187ec8e01bf7e6e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 15 Jul 2015 15:15:08 -0500 Subject: [PATCH 0260/2431] Whitelist murmur3 extension for PyPy runtime --- setup.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index bf12426231..d89a5ca52b 100644 --- a/setup.py +++ b/setup.py @@ -268,7 +268,7 @@ def run_setup(extensions): [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], extra_compile_args=compile_args) for m in cython_candidates], exclude_failures=True)) except ImportError: - print("Cython is not installed. Not compiling core driver files as extensions (optional).") + sys.stderr.write("Cython is not installed. Not compiling core driver files as extensions (optional).") if "--no-extensions" in sys.argv: extensions = [] @@ -290,14 +290,25 @@ def run_setup(extensions): =============================================================================== """ +pypy_unsupported_msg = \ +""" +================================================================================= +Some optional C extensions are not supported in PyPy. Only murmur3 will be built. +================================================================================= +""" + if extensions: - if (sys.platform.startswith("java") - or sys.platform == "cli"): + if "PyPy" in sys.version: + sys.stderr.write(pypy_unsupported_msg) + extensions = [ext for ext in extensions if ext is murmur3_ext] + + if (sys.platform.startswith("java") or sys.platform == "cli"): sys.stderr.write(platform_unsupported_msg) - extensions = () + extensions = [] elif sys.byteorder == "big": sys.stderr.write(arch_unsupported_msg) - extensions = () + extensions = [] + while True: # try to build as many of the extensions as we can From 5391a206343deb129db34a252348f56f8c45a451 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 10 Jul 2015 09:52:48 -0500 Subject: [PATCH 0261/2431] Limit mock dependency to version 1.0.1 1.1.0 dropped Python 2.6 support, and at least one call assert function currently used by tests. --- setup.py | 2 +- test-requirements.txt | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d89a5ca52b..37899c2e01 100644 --- a/setup.py +++ b/setup.py @@ -216,7 +216,7 @@ def run_setup(extensions): keywords='cassandra,cql,orm', include_package_data=True, install_requires=dependencies, - tests_require=['nose', 'mock', 'PyYAML', 'pytz', 'sure'], + tests_require=['nose', 'mock<=1.0.1', 'PyYAML', 'pytz', 'sure'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/test-requirements.txt b/test-requirements.txt index a90f1ad5c5..cc7b3c2b2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ blist scales nose -mock +mock<=1.0.1 ccm>=2.0 unittest2 PyYAML diff --git a/tox.ini b/tox.ini index b1f99087ad..d1ffc5456d 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py{26,27,33,34},pypy,pypy3 [base] deps = nose - mock + mock<=1.0.1 PyYAML six From be47fce1c13a2c574a07bdef5dcead2d966acac7 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 15 Jul 2015 15:29:23 -0500 Subject: [PATCH 0262/2431] Make tox pypy commands use absolute path to unit tests follows from new setdir --- tox.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index d1ffc5456d..5b5832af0c 100644 --- a/tox.ini +++ b/tox.ini @@ -20,13 +20,11 @@ commands = nosetests --verbosity=2 --no-path-adjustment {toxinidir}/tests/unit/ [testenv:pypy] deps = {[base]deps} -commands = {envpython} setup.py build_ext --inplace - nosetests --verbosity=2 tests/unit/ +commands = nosetests --verbosity=2 {toxinidir}/tests/unit/ [testenv:pypy3] deps = {[base]deps} -commands = {envpython} setup.py build_ext --inplace - nosetests --verbosity=2 tests/unit/ +commands = nosetests --verbosity=2 {toxinidir}/tests/unit/ # cqlengine/test_timestamp.py uses sure, which fails in pypy presently # could remove sure usage From 0d9cfbd77c2035ce091b032362ab33121cb938d1 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 15 Jul 2015 16:47:55 -0500 Subject: [PATCH 0263/2431] Removed unused cassandra.protocol._message_types_by_name --- cassandra/protocol.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 0aa9ae4846..0dbd513e65 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -64,7 +64,6 @@ class InternalError(Exception): CUSTOM_PAYLOAD_FLAG = 0x04 WARNING_FLAG = 0x08 -_message_types_by_name = {} _message_types_by_opcode = {} _UNSET_VALUE = object() @@ -72,7 +71,6 @@ class InternalError(Exception): class _RegisterMessageType(type): def __init__(cls, name, bases, dct): if not name.startswith('_'): - _message_types_by_name[cls.name] = cls _message_types_by_opcode[cls.opcode] = cls From d5b7f9264295dd16345e09e843f2ca1fdb4428b5 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Jul 2015 11:16:17 -0500 Subject: [PATCH 0264/2431] Add pypy3 config to .travis.yml --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3aa6228b92..dea9855234 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,15 +7,17 @@ env: - TOX_ENV=py27 CASS_VER=20 - TOX_ENV=py27 CASS_VER=21 - TOX_ENV=py27 CASS_VER=21 CYTHON_DEP=cython - - TOX_ENV=pypy CASS_VER=21 - TOX_ENV=py33 CASS_VER=21 - TOX_ENV=py33 CASS_VER=21 CYTHON_DEP=cython - TOX_ENV=py34 CASS_VER=21 - TOX_ENV=py34 CASS_VER=21 CYTHON_DEP=cython + - TOX_ENV=pypy CASS_VER=21 + - TOX_ENV=pypy3 CASS_VER=21 before_install: - sudo apt-get update -y - sudo apt-get install -y build-essential python-dev + - sudo apt-get install -y pypy-dev - sudo apt-get install -y libev4 libev-dev - sudo echo "deb http://www.apache.org/dist/cassandra/debian ${CASS_VER}x main" | sudo tee -a /etc/apt/sources.list - sudo echo "deb-src http://www.apache.org/dist/cassandra/debian ${CASS_VER}x main" | sudo tee -a /etc/apt/sources.list From ef07a9ec4826cc950574d496fa65d5b4488d068a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Jul 2015 11:48:48 -0500 Subject: [PATCH 0265/2431] Type code "enum" in protocol --- cassandra/protocol.py | 58 ++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 0dbd513e65..41439334d9 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -531,6 +531,34 @@ def send_body(self, f, protocol_version): RESULT_KIND_PREPARED = 0x0004 RESULT_KIND_SCHEMA_CHANGE = 0x0005 +class CassandraTypeCodes(object): + CUSTOM_TYPE = 0x0000 + AsciiType = 0x0001 + LongType = 0x0002 + BytesType = 0x0003 + BooleanType = 0x0004 + CounterColumnType = 0x0005 + DecimalType = 0x0006 + DoubleType = 0x0007 + FloatType = 0x0008 + Int32Type = 0x0009 + UTF8Type = 0x000A + DateType = 0x000B + UUIDType = 0x000C + UTF8Type = 0x000D + IntegerType = 0x000E + TimeUUIDType = 0x000F + InetAddressType = 0x0010 + SimpleDateType = 0x0011 + TimeType = 0x0012 + ShortType = 0x0013 + ByteType = 0x0014 + ListType = 0x0020 + MapType = 0x0021 + SetType = 0x0022 + UserType = 0x0030 + TupleType = 0x0031 + class ResultMessage(_MessageType): opcode = 0x08 @@ -540,34 +568,8 @@ class ResultMessage(_MessageType): results = None paging_state = None - type_codes = { - 0x0000: CUSTOM_TYPE, - 0x0001: AsciiType, - 0x0002: LongType, - 0x0003: BytesType, - 0x0004: BooleanType, - 0x0005: CounterColumnType, - 0x0006: DecimalType, - 0x0007: DoubleType, - 0x0008: FloatType, - 0x0009: Int32Type, - 0x000A: UTF8Type, - 0x000B: DateType, - 0x000C: UUIDType, - 0x000D: UTF8Type, - 0x000E: IntegerType, - 0x000F: TimeUUIDType, - 0x0010: InetAddressType, - 0x0011: SimpleDateType, - 0x0012: TimeType, - 0x0013: ShortType, - 0x0014: ByteType, - 0x0020: ListType, - 0x0021: MapType, - 0x0022: SetType, - 0x0030: UserType, - 0x0031: TupleType, - } + # Names match type name in module scope. Most are imported from cassandra.cqltypes (except CUSTOM_TYPE) + type_codes = _cqltypes_by_code = dict((v, globals()[k]) for k, v in CassandraTypeCodes.__dict__.items() if not k.startswith('_')) _FLAGS_GLOBAL_TABLES_SPEC = 0x0001 _HAS_MORE_PAGES_FLAG = 0x0002 From c9fb54ada1df6a7324c858818182ad04b542fb0a Mon Sep 17 00:00:00 2001 From: GregBestland Date: Thu, 16 Jul 2015 11:49:58 -0500 Subject: [PATCH 0266/2431] Deeply nested tuples can cause a stack overflow on the C* when using small stack sizes. This manifested it's self with Java 8. Reducing the depth of nested tuples in the test to ensure we don't run into this problem. --- tests/integration/standard/test_types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index f5c148ddd4..7321ef1e00 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -627,13 +627,13 @@ def test_can_insert_nested_tuples(self): "v_1 frozen<%s>," "v_2 frozen<%s>," "v_3 frozen<%s>," - "v_128 frozen<%s>" + "v_32 frozen<%s>" ")" % (self.nested_tuples_schema_helper(1), self.nested_tuples_schema_helper(2), self.nested_tuples_schema_helper(3), - self.nested_tuples_schema_helper(128))) + self.nested_tuples_schema_helper(32))) - for i in (1, 2, 3, 128): + for i in (1, 2, 3, 32): # create tuple created_tuple = self.nested_tuples_creator_helper(i) From 7dcabd744a419dd9b3b9345d3854373e0ade37e6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Jul 2015 14:35:21 -0500 Subject: [PATCH 0267/2431] Use real locks, not mocks, when dealing with threads in unit tests --- tests/unit/test_host_connection_pool.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_host_connection_pool.py b/tests/unit/test_host_connection_pool.py index 7a351afb2e..fec889e922 100644 --- a/tests/unit/test_host_connection_pool.py +++ b/tests/unit/test_host_connection_pool.py @@ -18,7 +18,7 @@ import unittest # noqa from mock import Mock, NonCallableMagicMock -from threading import Thread, Event +from threading import Thread, Event, Lock from cassandra.cluster import Session from cassandra.connection import Connection @@ -74,7 +74,7 @@ def test_failed_wait_for_connection(self): def test_successful_wait_for_connection(self): host = Mock(spec=Host, address='ip1') session = self.make_session() - conn = NonCallableMagicMock(spec=Connection, in_flight=0, is_defunct=False, is_closed=False, max_request_id=100) + conn = NonCallableMagicMock(spec=Connection, in_flight=0, is_defunct=False, is_closed=False, max_request_id=100, lock=Lock()) session.cluster.connection_factory.return_value = conn pool = HostConnectionPool(host, HostDistance.LOCAL, session) @@ -98,7 +98,7 @@ def get_second_conn(): def test_all_connections_trashed(self): host = Mock(spec=Host, address='ip1') session = self.make_session() - conn = NonCallableMagicMock(spec=Connection, in_flight=0, is_defunct=False, is_closed=False, max_request_id=100) + conn = NonCallableMagicMock(spec=Connection, in_flight=0, is_defunct=False, is_closed=False, max_request_id=100, lock=Lock()) session.cluster.connection_factory.return_value = conn session.cluster.get_core_connections_per_host.return_value = 1 From ab9f690707b6a1a76dbd6521e907bfdf7283d8c4 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Jul 2015 15:13:19 -0500 Subject: [PATCH 0268/2431] Isolate custom protocol handling to client requests. --- cassandra/cluster.py | 38 +++++++++++++++------------- cassandra/connection.py | 33 ++++++++++++------------ cassandra/io/asyncorereactor.py | 4 +-- cassandra/io/eventletreactor.py | 2 +- cassandra/io/geventreactor.py | 2 +- cassandra/io/libevreactor.py | 2 +- cassandra/io/twistedreactor.py | 6 ++--- tests/unit/io/test_twistedreactor.py | 4 +-- tests/unit/test_connection.py | 16 ++++++------ tests/unit/test_response_future.py | 12 ++++----- 10 files changed, 61 insertions(+), 58 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index c26e10f1a5..5f25e1781c 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -420,15 +420,6 @@ def auth_provider(self, value): GeventConnection will be used automatically. """ - protocol_handler_class = ProtocolHandler - """ - Specifies a protocol handler class, which can be used to override or extend features - such as message or type deserialization. - - The class must conform to the public classmethod interface defined in the default - implementation, :class:`cassandra.protocol.ProtocolHandler` - """ - control_connection_timeout = 2.0 """ A timeout, in seconds, for queries made by the control connection, such @@ -525,8 +516,7 @@ def __init__(self, idle_heartbeat_interval=30, schema_event_refresh_window=2, topology_event_refresh_window=10, - connect_timeout=5, - protocol_handler_class=None): + connect_timeout=5): """ Any of the mutable Cluster attributes may be set as keyword arguments to the constructor. @@ -570,9 +560,6 @@ def __init__(self, if connection_class is not None: self.connection_class = connection_class - if protocol_handler_class is not None: - self.protocol_handler_class = protocol_handler_class - self.metrics_enabled = metrics_enabled self.ssl_options = ssl_options self.sockopts = sockopts @@ -812,7 +799,6 @@ def _make_connection_kwargs(self, address, kwargs_dict): kwargs_dict.setdefault('cql_version', self.cql_version) kwargs_dict.setdefault('protocol_version', self.protocol_version) kwargs_dict.setdefault('user_type_map', self._user_types) - kwargs_dict.setdefault('protocol_handler_class', self.protocol_handler_class) return kwargs_dict @@ -1372,7 +1358,7 @@ def _prepare_all_queries(self, host): log.debug("Preparing all known prepared statements against host %s", host) connection = None try: - connection = self.connection_factory(host.address, protocol_handler_class=ProtocolHandler) + connection = self.connection_factory(host.address) try: self.control_connection.wait_for_schema_agreement(connection) except Exception: @@ -1535,6 +1521,20 @@ class Session(object): .. versionadded:: 2.1.0 """ + client_protocol_handler = ProtocolHandler + """ + Specifies a protocol handler that will be used for client-initiated requests (i.e. no + internal driver requests). This can be used to override or extend features such as + message or type ser/des. + + The class must conform to the public classmethod interface defined in the default + implementation, :class:`cassandra.protocol.ProtocolHandler` + + This is not included in published documentation as it is not intended for the casual user. + It requires knowledge of the native protocol and driver internals. Only advanced, specialized + use cases should need to do anything with this. + """ + _lock = None _pools = None _load_balancer = None @@ -1661,6 +1661,7 @@ def execute_async(self, query, parameters=None, trace=False, custom_payload=None timeout = self.default_timeout future = self._create_response_future(query, parameters, trace, custom_payload, timeout) + future._protocol_handler = self.client_protocol_handler future.send_request() return future @@ -2131,7 +2132,7 @@ def _try_connect(self, host): while True: try: - connection = self._cluster.connection_factory(host.address, is_control_connection=True, protocol_handler_class=ProtocolHandler) + connection = self._cluster.connection_factory(host.address, is_control_connection=True) break except ProtocolVersionUnsupported as e: self._cluster.protocol_downgrade(host.address, e.startup_version) @@ -2846,6 +2847,7 @@ class ResponseFuture(object): _custom_payload = None _warnings = None _timer = None + _protocol_handler = ProtocolHandler def __init__(self, session, message, query, timeout, metrics=None, prepared_statement=None): self.session = session @@ -2919,7 +2921,7 @@ def _query(self, host, message=None, cb=None): # TODO get connectTimeout from cluster settings connection, request_id = pool.borrow_connection(timeout=2.0) self._connection = connection - connection.send_msg(message, request_id, cb=cb) + connection.send_msg(message, request_id, cb=cb, encoder=self._protocol_handler.encode_message, decoder=self._protocol_handler.decode_message) return request_id except NoConnectionsAvailable as exc: log.debug("All connections for host %s are at capacity, moving to the next host", host) diff --git a/cassandra/connection.py b/cassandra/connection.py index d1e3a9c97a..5db88985e3 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -209,7 +209,7 @@ class Connection(object): def __init__(self, host='127.0.0.1', port=9042, authenticator=None, ssl_options=None, sockopts=None, compression=True, cql_version=None, protocol_version=MAX_SUPPORTED_VERSION, is_control_connection=False, - user_type_map=None, protocol_handler_class=ProtocolHandler): + user_type_map=None): self.host = host self.port = port self.authenticator = authenticator @@ -220,10 +220,8 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.protocol_version = protocol_version self.is_control_connection = is_control_connection self.user_type_map = user_type_map - self.decoder = protocol_handler_class.decode_message - self.encoder = protocol_handler_class.encode_message self._push_watchers = defaultdict(set) - self._callbacks = {} + self._requests = {} self._iobuf = io.BytesIO() if protocol_version >= 3: @@ -320,20 +318,20 @@ def defunct(self, exc): self.last_error = exc self.close() - self.error_all_callbacks(exc) + self.error_all_requests(exc) self.connected_event.set() return exc - def error_all_callbacks(self, exc): + def error_all_requests(self, exc): with self.lock: - callbacks = self._callbacks - self._callbacks = {} + requests = self._requests + self._requests = {} new_exc = ConnectionShutdown(str(exc)) - for cb in callbacks.values(): + for cb, _ in requests.values(): try: cb(new_exc) except Exception: - log.warning("Ignoring unhandled exception while erroring callbacks for a " + log.warning("Ignoring unhandled exception while erroring requests for a " "failed connection (%s) to host %s:", id(self), self.host, exc_info=True) @@ -357,14 +355,16 @@ def handle_pushed(self, response): except Exception: log.exception("Pushed event handler errored, ignoring:") - def send_msg(self, msg, request_id, cb): + def send_msg(self, msg, request_id, cb, encoder=ProtocolHandler.encode_message, decoder=ProtocolHandler.decode_message): if self.is_defunct: raise ConnectionShutdown("Connection to %s is defunct" % self.host) elif self.is_closed: raise ConnectionShutdown("Connection to %s is closed" % self.host) - self._callbacks[request_id] = cb - self.push(self.encoder(msg, request_id, self.protocol_version, compressor=self.compressor)) + # queue the decoder function with the request + # this allows us to inject custom functions per request to encode, decode messages + self._requests[request_id] = (cb, decoder) + self.push(encoder(msg, request_id, self.protocol_version, compressor=self.compressor)) return request_id def wait_for_response(self, msg, timeout=None): @@ -492,16 +492,17 @@ def process_msg(self, header, body): stream_id = header.stream if stream_id < 0: callback = None + decoder = ProtocolHandler.decode_message else: - callback = self._callbacks.pop(stream_id, None) + callback, decoder = self._requests.pop(stream_id, None) with self.lock: self.request_ids.append(stream_id) self.msg_received = True try: - response = self.decoder(header.version, self.user_type_map, stream_id, - header.flags, header.opcode, body, self.decompressor) + response = decoder(header.version, self.user_type_map, stream_id, + header.flags, header.opcode, body, self.decompressor) except Exception as exc: log.exception("Error decoding response from Cassandra. " "opcode: %04x; message contents: %r", header.opcode, body) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index fae87a7354..dc9d26c6c9 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -183,7 +183,7 @@ def close(self): log.debug("Closed socket to %s", self.host) if not self.is_defunct: - self.error_all_callbacks( + self.error_all_requests( ConnectionShutdown("Connection to %s was closed" % self.host)) # don't leave in-progress operations hanging self.connected_event.set() @@ -239,7 +239,7 @@ def handle_read(self): if self._iobuf.tell(): self.process_io_buffer() - if not self._callbacks and not self.is_control_connection: + if not self._requests and not self.is_control_connection: self._readable = False def push(self, data): diff --git a/cassandra/io/eventletreactor.py b/cassandra/io/eventletreactor.py index b0206f43d2..aa90cc9abd 100644 --- a/cassandra/io/eventletreactor.py +++ b/cassandra/io/eventletreactor.py @@ -116,7 +116,7 @@ def close(self): log.debug("Closed socket to %s" % (self.host,)) if not self.is_defunct: - self.error_all_callbacks( + self.error_all_requests( ConnectionShutdown("Connection to %s was closed" % self.host)) # don't leave in-progress operations hanging self.connected_event.set() diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 60a3f2d031..f26e61523c 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -106,7 +106,7 @@ def close(self): log.debug("Closed socket to %s" % (self.host,)) if not self.is_defunct: - self.error_all_callbacks( + self.error_all_requests( ConnectionShutdown("Connection to %s was closed" % self.host)) # don't leave in-progress operations hanging self.connected_event.set() diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 9114af133b..6b2036e2ee 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -290,7 +290,7 @@ def close(self): # don't leave in-progress operations hanging if not self.is_defunct: - self.error_all_callbacks( + self.error_all_requests( ConnectionShutdown("Connection to %s was closed" % self.host)) def handle_write(self, watcher, revents, errno=None): diff --git a/cassandra/io/twistedreactor.py b/cassandra/io/twistedreactor.py index b02fb6ab32..967a968f24 100644 --- a/cassandra/io/twistedreactor.py +++ b/cassandra/io/twistedreactor.py @@ -96,7 +96,7 @@ def clientConnectionLost(self, connector, reason): It should be safe to call defunct() here instead of just close, because we can assume that if the connection was closed cleanly, there are no - callbacks to error out. If this assumption turns out to be false, we + requests to error out. If this assumption turns out to be false, we can call close() instead of defunct() when "reason" is an appropriate type. """ @@ -213,7 +213,7 @@ def client_connection_made(self): def close(self): """ - Disconnect and error-out all callbacks. + Disconnect and error-out all requests. """ with self.lock: if self.is_closed: @@ -225,7 +225,7 @@ def close(self): log.debug("Closed socket to %s", self.host) if not self.is_defunct: - self.error_all_callbacks( + self.error_all_requests( ConnectionShutdown("Connection to %s was closed" % self.host)) # don't leave in-progress operations hanging self.connected_event.set() diff --git a/tests/unit/io/test_twistedreactor.py b/tests/unit/io/test_twistedreactor.py index d2142b09ca..928436e060 100644 --- a/tests/unit/io/test_twistedreactor.py +++ b/tests/unit/io/test_twistedreactor.py @@ -140,13 +140,13 @@ def test_close(self, mock_connectTCP): """ Verify that close() disconnects the connector and errors callbacks. """ - self.obj_ut.error_all_callbacks = Mock() + self.obj_ut.error_all_requests = Mock() self.obj_ut.add_connection() self.obj_ut.is_closed = False self.obj_ut.close() self.obj_ut.connector.disconnect.assert_called_with() self.assertTrue(self.obj_ut.connected_event.is_set()) - self.assertTrue(self.obj_ut.error_all_callbacks.called) + self.assertTrue(self.obj_ut.error_all_requests.called) def test_handle_read__incomplete(self): """ diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 268e19d535..521fab6105 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -27,7 +27,7 @@ locally_supported_compressions, ConnectionHeartbeat, _Frame) from cassandra.marshal import uint8_pack, uint32_pack, int32_pack from cassandra.protocol import (write_stringmultimap, write_int, write_string, - SupportedMessage) + SupportedMessage, ProtocolHandler) class ConnectionTest(unittest.TestCase): @@ -75,7 +75,7 @@ def make_msg(self, header, body=""): def test_bad_protocol_version(self, *args): c = self.make_connection() - c._callbacks = Mock() + c._requests = Mock() c.defunct = Mock() # read in a SupportedMessage response @@ -93,7 +93,7 @@ def test_bad_protocol_version(self, *args): def test_negative_body_length(self, *args): c = self.make_connection() - c._callbacks = Mock() + c._requests = Mock() c.defunct = Mock() # read in a SupportedMessage response @@ -110,7 +110,7 @@ def test_negative_body_length(self, *args): def test_unsupported_cql_version(self, *args): c = self.make_connection() - c._callbacks = {0: c._handle_options_response} + c._requests = {0: (c._handle_options_response, ProtocolHandler.decode_message)} c.defunct = Mock() c.cql_version = "3.0.3" @@ -133,7 +133,7 @@ def test_unsupported_cql_version(self, *args): def test_prefer_lz4_compression(self, *args): c = self.make_connection() - c._callbacks = {0: c._handle_options_response} + c._requests = {0: (c._handle_options_response, ProtocolHandler.decode_message)} c.defunct = Mock() c.cql_version = "3.0.3" @@ -156,7 +156,7 @@ def test_prefer_lz4_compression(self, *args): def test_requested_compression_not_available(self, *args): c = self.make_connection() - c._callbacks = {0: c._handle_options_response} + c._requests = {0: (c._handle_options_response, ProtocolHandler.decode_message)} c.defunct = Mock() # request lz4 compression c.compression = "lz4" @@ -186,7 +186,7 @@ def test_requested_compression_not_available(self, *args): def test_use_requested_compression(self, *args): c = self.make_connection() - c._callbacks = {0: c._handle_options_response} + c._requests = {0: (c._handle_options_response, ProtocolHandler.decode_message)} c.defunct = Mock() # request snappy compression c.compression = "snappy" @@ -213,7 +213,7 @@ def test_use_requested_compression(self, *args): def test_disable_compression(self, *args): c = self.make_connection() - c._callbacks = {0: c._handle_options_response} + c._requests = {0: (c._handle_options_response, ProtocolHandler.decode_message)} c.defunct = Mock() # disable compression c.compression = False diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index eea43f7586..d266ab774e 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -27,7 +27,7 @@ OverloadedErrorMessage, IsBootstrappingErrorMessage, PreparedQueryNotFound, PrepareMessage, RESULT_KIND_ROWS, RESULT_KIND_SET_KEYSPACE, - RESULT_KIND_SCHEMA_CHANGE) + RESULT_KIND_SCHEMA_CHANGE, ProtocolHandler) from cassandra.policies import RetryPolicy from cassandra.pool import NoConnectionsAvailable from cassandra.query import SimpleStatement @@ -67,7 +67,7 @@ def test_result_message(self): rf.session._pools.get.assert_called_once_with('ip1') pool.borrow_connection.assert_called_once_with(timeout=ANY) - connection.send_msg.assert_called_once_with(rf.message, 1, cb=ANY) + connection.send_msg.assert_called_once_with(rf.message, 1, cb=ANY, encoder=ProtocolHandler.encode_message, decoder=ProtocolHandler.decode_message) rf._set_result(self.make_mock_response([{'col': 'val'}])) result = rf.result() @@ -189,7 +189,7 @@ def test_retry_policy_says_retry(self): rf.session._pools.get.assert_called_once_with('ip1') pool.borrow_connection.assert_called_once_with(timeout=ANY) - connection.send_msg.assert_called_once_with(rf.message, 1, cb=ANY) + connection.send_msg.assert_called_once_with(rf.message, 1, cb=ANY, encoder=ProtocolHandler.encode_message, decoder=ProtocolHandler.decode_message) result = Mock(spec=UnavailableErrorMessage, info={}) rf._set_result(result) @@ -207,7 +207,7 @@ def test_retry_policy_says_retry(self): # an UnavailableException rf.session._pools.get.assert_called_with('ip1') pool.borrow_connection.assert_called_with(timeout=ANY) - connection.send_msg.assert_called_with(rf.message, 2, cb=ANY) + connection.send_msg.assert_called_with(rf.message, 2, cb=ANY, encoder=ProtocolHandler.encode_message, decoder=ProtocolHandler.decode_message) def test_retry_with_different_host(self): session = self.make_session() @@ -222,7 +222,7 @@ def test_retry_with_different_host(self): rf.session._pools.get.assert_called_once_with('ip1') pool.borrow_connection.assert_called_once_with(timeout=ANY) - connection.send_msg.assert_called_once_with(rf.message, 1, cb=ANY) + connection.send_msg.assert_called_once_with(rf.message, 1, cb=ANY, encoder=ProtocolHandler.encode_message, decoder=ProtocolHandler.decode_message) self.assertEqual(ConsistencyLevel.QUORUM, rf.message.consistency_level) result = Mock(spec=OverloadedErrorMessage, info={}) @@ -240,7 +240,7 @@ def test_retry_with_different_host(self): # it should try with a different host rf.session._pools.get.assert_called_with('ip2') pool.borrow_connection.assert_called_with(timeout=ANY) - connection.send_msg.assert_called_with(rf.message, 2, cb=ANY) + connection.send_msg.assert_called_with(rf.message, 2, cb=ANY, encoder=ProtocolHandler.encode_message, decoder=ProtocolHandler.decode_message) # the consistency level should be the same self.assertEqual(ConsistencyLevel.QUORUM, rf.message.consistency_level) From 49357dec114fad411f6b712dda4fa5ddc3e72736 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Thu, 16 Jul 2015 16:08:47 -0500 Subject: [PATCH 0269/2431] Simplify logic in Downgrading write timeout, and test --- cassandra/policies.py | 13 +++++++++---- tests/unit/test_policies.py | 15 ++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index 6d6c23b972..328b6b82fb 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -816,14 +816,19 @@ def on_write_timeout(self, query, consistency, write_type, required_responses, received_responses, retry_num): if retry_num != 0: return (self.RETHROW, None) - elif write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER) and received_responses > 0: - return (self.IGNORE, None) + + if write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER): + if received_responses > 0: + # persisted on at least one replica + return (self.IGNORE, None) + else: + return (self.RETHROW, None) elif write_type == WriteType.UNLOGGED_BATCH: return self._pick_consistency(received_responses) elif write_type == WriteType.BATCH_LOG: return (self.RETRY, consistency) - else: - return (self.RETHROW, None) + + return (self.RETHROW, None) def on_unavailable(self, query, consistency, required_replicas, alive_replicas, retry_num): if retry_num != 0: diff --git a/tests/unit/test_policies.py b/tests/unit/test_policies.py index bd8541047f..97e24455aa 100644 --- a/tests/unit/test_policies.py +++ b/tests/unit/test_policies.py @@ -1052,20 +1052,17 @@ def test_write_timeout(self): self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) - # On these type of writes failures should not be ignored - # if received_responses is 0 - for write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER): - retry, consistency = policy.on_write_timeout( - query=None, consistency=ONE, write_type=write_type, - required_responses=1, received_responses=0, retry_num=0) - self.assertEqual(retry, RetryPolicy.RETHROW) - - # ignore failures on these types of writes for write_type in (WriteType.SIMPLE, WriteType.BATCH, WriteType.COUNTER): + # ignore failures if at least one response (replica persisted) retry, consistency = policy.on_write_timeout( query=None, consistency=ONE, write_type=write_type, required_responses=1, received_responses=2, retry_num=0) self.assertEqual(retry, RetryPolicy.IGNORE) + # retrhow if we can't be sure we have a replica + retry, consistency = policy.on_write_timeout( + query=None, consistency=ONE, write_type=write_type, + required_responses=1, received_responses=0, retry_num=0) + self.assertEqual(retry, RetryPolicy.RETHROW) # downgrade consistency level on unlogged batch writes retry, consistency = policy.on_write_timeout( From 53c17166642c018446aa42d6eab2a9329fa9f184 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 17 Jul 2015 13:46:16 -0500 Subject: [PATCH 0270/2431] cqlengine example tweaks moved import, added a LWT example, set schema management var to avoid warning --- example_mapper.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/example_mapper.py b/example_mapper.py index eda77c2e73..31a39eec6d 100755 --- a/example_mapper.py +++ b/example_mapper.py @@ -14,10 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# silence warnings just for demo -- applications would typically not do this +import os +os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1' + import logging log = logging.getLogger() -log.setLevel('DEBUG') +log.setLevel('INFO') handler = logging.StreamHandler() handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")) log.addHandler(handler) @@ -27,9 +31,9 @@ from cassandra.cqlengine import columns from cassandra.cqlengine import connection from cassandra.cqlengine import management -from cassandra.cqlengine.exceptions import ValidationError +from cassandra.cqlengine import ValidationError from cassandra.cqlengine.models import Model -from cassandra.cqlengine.query import BatchQuery +from cassandra.cqlengine.query import BatchQuery, LWTException KEYSPACE = "testkeyspace" @@ -61,12 +65,19 @@ def main(): log.info("### syncing model...") management.sync_table(FamilyMembers) - log.info("### add entities serially") - simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m') # default uuid is assigned + # default uuid is assigned + simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m') - # add members later + # add members to his family later FamilyMembers.create(id=simmons.id, surname='Simmons', name='Nick', birth_year=1989, sex='m') - FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f') + sophie = FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f') + + nick = FamilyMembers.objects(id=simmons.id, surname='Simmons', name='Nick') + try: + nick.iff(birth_year=1988).update(birth_year=1989) + except LWTException: + print "precondition not met" + # showing validation try: FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', birth_year=1957, sex='f') @@ -95,6 +106,9 @@ def main(): for m in FamilyMembers.objects(id=simmons.id, surname=simmons.surname): print m, m.birth_year, m.sex + log.info("### Constrain on clustering key") + kids = FamilyMembers.objects(id=simmons.id, surname=simmons.surname, name__in=['Nick', 'Sophie']) + log.info("### Delete a record") FamilyMembers(id=hogan_id, surname='Hogan', name='Linda').delete() for m in FamilyMembers.objects(id=hogan_id): From 1a480f196ade42798596f5257d2cbeffcadf154f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 17 Jul 2015 16:14:30 -0500 Subject: [PATCH 0271/2431] Test updates removing refs to schema tables. --- tests/integration/standard/test_cluster.py | 6 +++--- tests/integration/standard/test_types.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index b8439c4de3..f218fd5d26 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -314,13 +314,13 @@ def test_refresh_schema_table(self): original_meta = cluster.metadata.keyspaces original_system_meta = original_meta['system'] - original_system_schema_meta = original_system_meta.tables['schema_columnfamilies'] + original_system_schema_meta = original_system_meta.tables['local'] # only refresh one table - cluster.refresh_table_metadata('system', 'schema_columnfamilies') + cluster.refresh_table_metadata('system', 'local') current_meta = cluster.metadata.keyspaces current_system_meta = current_meta['system'] - current_system_schema_meta = current_system_meta.tables['schema_columnfamilies'] + current_system_schema_meta = current_system_meta.tables['local'] self.assertIs(original_meta, current_meta) self.assertIs(original_system_meta, current_system_meta) self.assertIsNot(original_system_schema_meta, current_system_schema_meta) diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index f5c148ddd4..fa94b88ba4 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -677,9 +677,8 @@ def test_can_insert_unicode_query_string(self): Test to ensure unicode strings can be used in a query """ s = self.session - - query = u"SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = 'ef\u2052ef' AND columnfamily_name = %s" - s.execute(query, (u"fe\u2051fe",)) + s.execute(u"SELECT * FROM system.local WHERE key = 'ef\u2052ef'") + s.execute(u"SELECT * FROM system.local WHERE key = %s", (u"fe\u2051fe",)) def test_can_read_composite_type(self): """ From b95bb9fefd5b53d85eaf20c7f92dfd68108e6f9c Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 12:45:46 -0500 Subject: [PATCH 0272/2431] 2.6 changelog update --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a4111bd507..72949a1e02 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,11 @@ +2.6.0 +===== +July 20, 2015 + +Bug Fixes +--------- +* Output proper CQL for compact tables with no clustering columns (PYTHON-360) + 2.6.0c2 ======= June 24, 2015 From 536d7c36ee69ebb7273edfbe78babd40849e620a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 12:48:56 -0500 Subject: [PATCH 0273/2431] 2.6.0 release version --- cassandra/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 23d4623d24..0a0f29320f 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 6, '0c2', 'post') +__version_info__ = (2, 6, 0) __version__ = '.'.join(map(str, __version_info__)) From 64c95a7ab1453dd56759b08d42cd1ae8e0292f4a Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 13:54:23 -0500 Subject: [PATCH 0274/2431] doc: remove rtype from util functions was being used as x-ref and linking into cqlengine attributes --- cassandra/util.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cassandra/util.py b/cassandra/util.py index ba49b5687e..83260577df 100644 --- a/cassandra/util.py +++ b/cassandra/util.py @@ -15,8 +15,6 @@ def datetime_from_timestamp(timestamp): and rounding differences in Python 3.4 (PYTHON-340). :param timestamp: a unix timestamp, in seconds - - :rtype: datetime """ dt = DATETIME_EPOC + datetime.timedelta(seconds=timestamp) return dt @@ -29,9 +27,6 @@ def unix_time_from_uuid1(uuid_arg): results of queries returning a v1 :class:`~uuid.UUID`. :param uuid_arg: a version 1 :class:`~uuid.UUID` - - :rtype: timestamp - """ return (uuid_arg.time - 0x01B21DD213814000) / 1e7 @@ -42,9 +37,6 @@ def datetime_from_uuid1(uuid_arg): specified type-1 UUID. :param uuid_arg: a version 1 :class:`~uuid.UUID` - - :rtype: timestamp - """ return datetime_from_timestamp(unix_time_from_uuid1(uuid_arg)) From 0dfcbcabf6dbb720eb27fcdf8b84936571a8c0fc Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 14:15:39 -0500 Subject: [PATCH 0275/2431] post release version update --- cassandra/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 0a0f29320f..4bec0e8aad 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (2, 6, 0) +__version_info__ = (2, 6, 0, 'post') __version__ = '.'.join(map(str, __version_info__)) From f7ef27e006c72049fd99a25db61f8a31f1997907 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Thu, 2 Jul 2015 18:02:29 -0500 Subject: [PATCH 0276/2431] Adding timer tests for async, libev, and twisted. Added integration tests. All for PYTHON-108 --- tests/__init__.py | 18 ++-- tests/integration/long/test_failure_types.py | 90 +++++++++++++++- tests/unit/io/eventlet_utils.py | 37 +++++++ tests/unit/io/gevent_utils.py | 56 ++++++++++ tests/unit/io/test_asyncorereactor.py | 40 ++++++- tests/unit/io/test_eventletreactor.py | 67 ++++++++++++ tests/unit/io/test_geventreactor.py | 78 ++++++++++++++ tests/unit/io/test_libevreactor.py | 9 +- tests/unit/io/test_libevtimer.py | 82 ++++++++++++++ tests/unit/io/test_twistedreactor.py | 53 ++++++++- tests/unit/io/utils.py | 107 +++++++++++++++++++ tox.ini | 13 ++- 12 files changed, 632 insertions(+), 18 deletions(-) create mode 100644 tests/unit/io/eventlet_utils.py create mode 100644 tests/unit/io/gevent_utils.py create mode 100644 tests/unit/io/test_eventletreactor.py create mode 100644 tests/unit/io/test_geventreactor.py create mode 100644 tests/unit/io/test_libevtimer.py create mode 100644 tests/unit/io/utils.py diff --git a/tests/__init__.py b/tests/__init__.py index e1c1e54e11..150e600c91 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,6 +14,7 @@ import logging import sys +import socket log = logging.getLogger() log.setLevel('DEBUG') @@ -24,15 +25,18 @@ log.addHandler(handler) -def is_gevent_monkey_patched(): - return 'gevent.monkey' in sys.modules +def is_eventlet_monkey_patched(): + if 'eventlet.patcher' not in sys.modules: + return False + import eventlet.patcher + return eventlet.patcher.is_monkey_patched('socket') -def is_eventlet_monkey_patched(): - if 'eventlet.patcher' in sys.modules: - import eventlet - return eventlet.patcher.is_monkey_patched('socket') - return False +def is_gevent_monkey_patched(): + if 'gevent.monkey' not in sys.modules: + return False + import gevent.socket + return socket.socket is gevent.socket.socket def is_monkey_patched(): diff --git a/tests/integration/long/test_failure_types.py b/tests/integration/long/test_failure_types.py index 30e05b60b6..e611adca35 100644 --- a/tests/integration/long/test_failure_types.py +++ b/tests/integration/long/test_failure_types.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys, logging, traceback +import sys,logging, traceback, time from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, ReadFailure, WriteFailure,\ FunctionFailure from cassandra.cluster import Cluster from cassandra.concurrent import execute_concurrent_with_args from cassandra.query import SimpleStatement -from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace, remove_cluster +from tests.integration import use_singledc, PROTOCOL_VERSION, get_cluster, setup_keyspace, remove_cluster, get_node +from mock import Mock try: import unittest2 as unittest @@ -301,3 +302,88 @@ def test_user_function_failure(self): """ DROP TABLE test3rf.d; """, consistency_level=ConsistencyLevel.ALL, expected_exception=None) + + +class TimeoutTimerTest(unittest.TestCase): + def setUp(self): + """ + Setup sessions and pause node1 + """ + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + + # self.node1, self.node2, self.node3 = get_cluster().nodes.values() + self.node1 = get_node(1) + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + + ddl = ''' + CREATE TABLE test3rf.timeout ( + k int PRIMARY KEY, + v int )''' + self.session.execute(ddl) + self.node1.pause() + + def tearDown(self): + """ + Shutdown cluster and resume node1 + """ + self.node1.resume() + self.session.execute("DROP TABLE test3rf.timeout") + self.cluster.shutdown() + + def test_async_timeouts(self): + """ + Test to validate that timeouts are honored + + + Exercise the underlying timeouts, by attempting a query that will timeout. Ensure the default timeout is still + honored. Make sure that user timeouts are also honored. + + @since 2.7.0 + @jira_ticket PYTHON-108 + @expected_result timeouts should be honored + + @test_category + + """ + + # Because node1 is stopped these statements will all timeout + ss = SimpleStatement('SELECT * FROM test3rf.test', consistency_level=ConsistencyLevel.ALL) + + # Test with default timeout (should be 10) + start_time = time.time() + future = self.session.execute_async(ss) + with self.assertRaises(OperationTimedOut): + future.result() + end_time = time.time() + total_time = end_time-start_time + expected_time = self.session.default_timeout + # check timeout and ensure it's within a reasonable range + self.assertAlmostEqual(expected_time, total_time, delta=.05) + + # Test with user defined timeout (Should be 1) + start_time = time.time() + future = self.session.execute_async(ss, timeout=1) + mock_callback = Mock(return_value=None) + mock_errorback = Mock(return_value=None) + future.add_callback(mock_callback) + future.add_errback(mock_errorback) + + with self.assertRaises(OperationTimedOut): + future.result() + end_time = time.time() + total_time = end_time-start_time + expected_time = 1 + # check timeout and ensure it's within a reasonable range + self.assertAlmostEqual(expected_time, total_time, delta=.05) + self.assertTrue(mock_errorback.called) + self.assertFalse(mock_callback.called) + + + + + + + + diff --git a/tests/unit/io/eventlet_utils.py b/tests/unit/io/eventlet_utils.py new file mode 100644 index 0000000000..8aee030f27 --- /dev/null +++ b/tests/unit/io/eventlet_utils.py @@ -0,0 +1,37 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import select +import socket +import thread +import Queue +import threading +import __builtin__ +import ssl +import time + + +def eventlet_un_patch_all(): + """ + A method to unpatch eventlet monkey patching used for the reactor tests + """ + + # These are the modules that are loaded by eventlet we reload them all + modules_to_unpatch = [os, select, socket, thread, time, Queue, threading, ssl, __builtin__] + for to_unpatch in modules_to_unpatch: + reload(to_unpatch) + + diff --git a/tests/unit/io/gevent_utils.py b/tests/unit/io/gevent_utils.py new file mode 100644 index 0000000000..1c6e27adde --- /dev/null +++ b/tests/unit/io/gevent_utils.py @@ -0,0 +1,56 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from gevent import monkey + + +def gevent_un_patch_all(): + """ + A method to unpatch gevent libraries. These are unloaded + in the same order that gevent monkey patch loads theirs. + Order cannot be arbitrary. This is used in the unit tests to + un monkey patch gevent + """ + restore_saved_module("os") + restore_saved_module("time") + restore_saved_module("thread") + restore_saved_module("threading") + restore_saved_module("_threading_local") + restore_saved_module("stdin") + restore_saved_module("stdout") + restore_saved_module("socket") + restore_saved_module("select") + restore_saved_module("ssl") + restore_saved_module("subprocess") + + +def restore_saved_module(module): + """ + gevent monkey patch keeps a list of all patched modules. + This will restore the original ones + :param module: to unpatch + :return: + """ + + # Check the saved attributes in geven monkey patch + if not (module in monkey.saved): + return + _module = __import__(module) + + # If it exist unpatch it + for attr in monkey.saved[module]: + if hasattr(_module, attr): + setattr(_module, attr, monkey.saved[module][attr]) + diff --git a/tests/unit/io/test_asyncorereactor.py b/tests/unit/io/test_asyncorereactor.py index 9d97f4688a..16fbf2fb72 100644 --- a/tests/unit/io/test_asyncorereactor.py +++ b/tests/unit/io/test_asyncorereactor.py @@ -21,20 +21,20 @@ import errno import math +import time from mock import patch, Mock import os from six import BytesIO import socket from socket import error as socket_error - from cassandra.connection import (HEADER_DIRECTION_TO_CLIENT, - ConnectionException, ProtocolError) + ConnectionException, ProtocolError,Timer) from cassandra.io.asyncorereactor import AsyncoreConnection from cassandra.protocol import (write_stringmultimap, write_int, write_string, SupportedMessage, ReadyMessage, ServerError) from cassandra.marshal import uint8_pack, uint32_pack, int32_pack - from tests import is_monkey_patched +from tests.unit.io.utils import submit_and_wait_for_completion, TimerCallback class AsyncoreConnectionTest(unittest.TestCase): @@ -300,3 +300,37 @@ def test_partial_message_read(self, *args): self.assertTrue(c.connected_event.is_set()) self.assertFalse(c.is_defunct) + + def test_multi_timer_validation(self, *args): + """ + Verify that timer timeouts are honored appropriately + """ + c = self.make_connection() + # Tests timers submitted in order at various timeouts + submit_and_wait_for_completion(self, AsyncoreConnection, 0, 100, 1, 100) + # Tests timers submitted in reverse order at various timeouts + submit_and_wait_for_completion(self, AsyncoreConnection, 100, 0, -1, 100) + # Tests timers submitted in varying order at various timeouts + submit_and_wait_for_completion(self, AsyncoreConnection, 0, 100, 1, 100, True) + + def test_timer_cancellation(self): + """ + Verify that timer cancellation is honored + """ + + # Various lists for tracking callback stage + connection = self.make_connection() + timeout = .1 + callback = TimerCallback(timeout) + timer = connection.create_timer(timeout, callback.invoke) + timer.cancel() + # Release context allow for timer thread to run. + time.sleep(.2) + timer_manager = connection._loop._timers + # Assert that the cancellation was honored + self.assertFalse(timer_manager._queue) + self.assertFalse(timer_manager._new_timers) + self.assertFalse(callback.was_invoked()) + + + diff --git a/tests/unit/io/test_eventletreactor.py b/tests/unit/io/test_eventletreactor.py new file mode 100644 index 0000000000..9f071d3bec --- /dev/null +++ b/tests/unit/io/test_eventletreactor.py @@ -0,0 +1,67 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + +from tests.unit.io.utils import submit_and_wait_for_completion, TimerCallback +from tests import is_eventlet_monkey_patched +import time + +try: + from cassandra.io.eventletreactor import EventletConnection +except ImportError: + EventletConnection = None # noqa + + +class EventletTimerTest(unittest.TestCase): + + def setUp(self): + if EventletConnection is None: + raise unittest.SkipTest("Eventlet libraries not available") + if not is_eventlet_monkey_patched(): + raise unittest.SkipTest("Can't test eventlet without monkey patching") + EventletConnection.initialize_reactor() + + def test_multi_timer_validation(self, *args): + """ + Verify that timer timeouts are honored appropriately + """ + # Tests timers submitted in order at various timeouts + submit_and_wait_for_completion(self, EventletConnection, 0, 100, 1, 100) + # Tests timers submitted in reverse order at various timeouts + submit_and_wait_for_completion(self, EventletConnection, 100, 0, -1, 100) + # Tests timers submitted in varying order at various timeouts + submit_and_wait_for_completion(self, EventletConnection, 0, 100, 1, 100, True) + + def test_timer_cancellation(self): + """ + Verify that timer cancellation is honored + """ + + # Various lists for tracking callback stage + timeout = .1 + callback = TimerCallback(timeout) + timer = EventletConnection.create_timer(timeout, callback.invoke) + timer.cancel() + # Release context allow for timer thread to run. + time.sleep(.2) + timer_manager = EventletConnection._timers + # Assert that the cancellation was honored + self.assertFalse(timer_manager._queue) + self.assertFalse(timer_manager._new_timers) + self.assertFalse(callback.was_invoked()) \ No newline at end of file diff --git a/tests/unit/io/test_geventreactor.py b/tests/unit/io/test_geventreactor.py new file mode 100644 index 0000000000..c90be37d24 --- /dev/null +++ b/tests/unit/io/test_geventreactor.py @@ -0,0 +1,78 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + +import time +from tests.unit.io.utils import submit_and_wait_for_completion, TimerCallback +from tests import is_gevent_monkey_patched + +try: + from cassandra.io.geventreactor import GeventConnection + import gevent.monkey + from gevent_utils import gevent_un_patch_all +except ImportError: + GeventConnection = None # noqa + + +class GeventTimerTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + if GeventConnection is not None: + if not is_gevent_monkey_patched(): + gevent.monkey.patch_all() + + @classmethod + def tearDownClass(cls): + if is_gevent_monkey_patched(): + gevent_un_patch_all() + + def setUp(self): + if not is_gevent_monkey_patched(): + raise unittest.SkipTest("Can't test gevent without monkey patching") + GeventConnection.initialize_reactor() + + def test_multi_timer_validation(self, *args): + """ + Verify that timer timeouts are honored appropriately + """ + + # Tests timers submitted in order at various timeouts + submit_and_wait_for_completion(self, GeventConnection, 0, 100, 1, 100) + # Tests timers submitted in reverse order at various timeouts + submit_and_wait_for_completion(self, GeventConnection, 100, 0, -1, 100) + # Tests timers submitted in varying order at various timeouts + submit_and_wait_for_completion(self, GeventConnection, 0, 100, 1, 100, True), + + def test_timer_cancellation(self): + """ + Verify that timer cancellation is honored + """ + + # Various lists for tracking callback stage + timeout = .1 + callback = TimerCallback(timeout) + timer = GeventConnection.create_timer(timeout, callback.invoke) + timer.cancel() + # Release context allow for timer thread to run. + time.sleep(.2) + timer_manager = GeventConnection._timers + # Assert that the cancellation was honored + self.assertFalse(timer_manager._queue) + self.assertFalse(timer_manager._new_timers) + self.assertFalse(callback.was_invoked()) diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index 334480ef3b..2b26343a63 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -24,6 +24,7 @@ from six import BytesIO from socket import error as socket_error import sys +import time from cassandra.connection import (HEADER_DIRECTION_TO_CLIENT, ConnectionException, ProtocolError) @@ -31,6 +32,10 @@ from cassandra.protocol import (write_stringmultimap, write_int, write_string, SupportedMessage, ReadyMessage, ServerError) from cassandra.marshal import uint8_pack, uint32_pack, int32_pack +from tests.unit.io.utils import TimerCallback +from tests.unit.io.utils import submit_and_wait_for_completion +from tests import is_monkey_patched + try: from cassandra.io.libevreactor import LibevConnection @@ -46,7 +51,7 @@ class LibevConnectionTest(unittest.TestCase): def setUp(self): - if 'gevent.monkey' in sys.modules: + if is_monkey_patched(): raise unittest.SkipTest("Can't test libev with monkey patching") if LibevConnection is None: raise unittest.SkipTest('libev does not appear to be installed correctly') @@ -290,4 +295,4 @@ def test_partial_message_read(self, *args): c.handle_read(None, 0) self.assertTrue(c.connected_event.is_set()) - self.assertFalse(c.is_defunct) + self.assertFalse(c.is_defunct) \ No newline at end of file diff --git a/tests/unit/io/test_libevtimer.py b/tests/unit/io/test_libevtimer.py new file mode 100644 index 0000000000..988282c2b1 --- /dev/null +++ b/tests/unit/io/test_libevtimer.py @@ -0,0 +1,82 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + + +from mock import patch, Mock + +import time + +from tests.unit.io.utils import submit_and_wait_for_completion, TimerCallback +from tests import is_monkey_patched + + +try: + from cassandra.io.libevreactor import LibevConnection +except ImportError: + LibevConnection = None # noqa + + +@patch('socket.socket') +@patch('cassandra.io.libevwrapper.IO') +class LibevTimerTest(unittest.TestCase): + + def setUp(self): + if is_monkey_patched(): + raise unittest.SkipTest("Can't test libev with monkey patching") + if LibevConnection is None: + raise unittest.SkipTest('libev does not appear to be installed correctly') + LibevConnection.initialize_reactor() + + def make_connection(self): + c = LibevConnection('1.2.3.4', cql_version='3.0.1') + c._socket = Mock() + c._socket.send.side_effect = lambda x: len(x) + return c + + def test_multi_timer_validation(self, *args): + """ + Verify that timer timeouts are honored appropriately + """ + c = self.make_connection() + c.initialize_reactor() + # Tests timers submitted in order at various timeouts + submit_and_wait_for_completion(self, c, 0, 100, 1, 100) + # Tests timers submitted in reverse order at various timeouts + submit_and_wait_for_completion(self, c, 100, 0, -1, 100) + # Tests timers submitted in varying order at various timeouts + submit_and_wait_for_completion(self, c, 0, 100, 1, 100, True) + + def test_timer_cancellation(self, *args): + """ + Verify that timer cancellation is honored + """ + + # Various lists for tracking callback stage + connection = self.make_connection() + timeout = .1 + callback = TimerCallback(timeout) + timer = connection.create_timer(timeout, callback.invoke) + timer.cancel() + # Release context allow for timer thread to run. + time.sleep(.2) + timer_manager = connection._libevloop._timers + # Assert that the cancellation was honored + self.assertFalse(timer_manager._queue) + self.assertFalse(timer_manager._new_timers) + self.assertFalse(callback.was_invoked()) + diff --git a/tests/unit/io/test_twistedreactor.py b/tests/unit/io/test_twistedreactor.py index d2142b09ca..7dcfa5609d 100644 --- a/tests/unit/io/test_twistedreactor.py +++ b/tests/unit/io/test_twistedreactor.py @@ -17,6 +17,7 @@ except ImportError: import unittest from mock import Mock, patch +import time try: from twisted.test import proto_helpers @@ -26,6 +27,54 @@ twistedreactor = None # NOQA from cassandra.connection import _Frame +from tests.unit.io.utils import submit_and_wait_for_completion, TimerCallback + + +class TestTwistedTimer(unittest.TestCase): + """ + Simple test class that is used to validate that the TimerManager, and timer + classes function appropriately with the twisted infrastructure + """ + + def setUp(self): + if twistedreactor is None: + raise unittest.SkipTest("Twisted libraries not available") + twistedreactor.TwistedConnection.initialize_reactor() + + def test_multi_timer_validation(self): + """ + Verify that the timers are called in the correct order + """ + twistedreactor.TwistedConnection.initialize_reactor() + connection = twistedreactor.TwistedConnection('1.2.3.4', + cql_version='3.0.1') + # Tests timers submitted in order at various timeouts + submit_and_wait_for_completion(self, connection, 0, 100, 1, 100) + # Tests timers submitted in reverse order at various timeouts + submit_and_wait_for_completion(self, connection, 100, 0, -1, 100) + # Tests timers submitted in varying order at various timeouts + submit_and_wait_for_completion(self, connection, 0, 100, 1, 100, True) + + def test_timer_cancellation(self, *args): + """ + Verify that timer cancellation is honored + """ + + # Various lists for tracking callback stage + connection = twistedreactor.TwistedConnection('1.2.3.4', + cql_version='3.0.1') + timeout = .1 + callback = TimerCallback(timeout) + timer = connection.create_timer(timeout, callback.invoke) + timer.cancel() + # Release context allow for timer thread to run. + time.sleep(.2) + timer_manager = connection._loop._timers + # Assert that the cancellation was honored + self.assertFalse(timer_manager._queue) + self.assertFalse(timer_manager._new_timers) + self.assertFalse(callback.was_invoked()) + class TestTwistedProtocol(unittest.TestCase): @@ -96,8 +145,6 @@ def setUp(self): twistedreactor.TwistedConnection.initialize_reactor() self.reactor_cft_patcher = patch( 'twisted.internet.reactor.callFromThread') - self.reactor_running_patcher = patch( - 'twisted.internet.reactor.running', False) self.reactor_run_patcher = patch('twisted.internet.reactor.run') self.mock_reactor_cft = self.reactor_cft_patcher.start() self.mock_reactor_run = self.reactor_run_patcher.start() @@ -107,7 +154,6 @@ def setUp(self): def tearDown(self): self.reactor_cft_patcher.stop() self.reactor_run_patcher.stop() - self.obj_ut._loop._cleanup() def test_connection_initialization(self): """ @@ -196,3 +242,4 @@ def test_push(self, mock_connectTCP): self.obj_ut.push('123 pickup') self.mock_reactor_cft.assert_called_with( self.obj_ut.connector.transport.write, '123 pickup') + diff --git a/tests/unit/io/utils.py b/tests/unit/io/utils.py new file mode 100644 index 0000000000..26307ffaf9 --- /dev/null +++ b/tests/unit/io/utils.py @@ -0,0 +1,107 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + + +class TimerCallback(): + + invoked = False + created_time = 0 + invoked_time = 0 + expected_wait = 0 + + def __init__(self, expected_wait): + self.invoked = False + self.created_time = time.time() + self.expected_wait = expected_wait + + def invoke(self): + self.invoked_time = time.time() + self.invoked = True + + def was_invoked(self): + return self.invoked + + def get_wait_time(self): + elapsed_time = self.invoked_time - self.created_time + return elapsed_time + + def wait_match_excepted(self): + if self.expected_wait-.01 <= self.get_wait_time() <= self.expected_wait+.01: + return True + return False + + +def get_timeout(gross_time, start, end, precision, split_range): + """ + A way to generate varying timeouts based on ranges + :param gross_time: Some integer between start and end + :param start: the start value of the range + :param end: the end value of the range + :param precision: the precision to use to generate the timeout. + :param split_range: generate values from both ends + :return: a timeout value to use + """ + if(split_range): + top_num = float(end)/precision + bottom_num = float(start)/precision + if gross_time % 2 == 0: + timeout = top_num - float(gross_time)/precision + else: + timeout = bottom_num + float(gross_time)/precision + + else: + timeout = float(gross_time)/precision + + return timeout + + +def submit_and_wait_for_completion(unit_test, connection, start, end, increment, precision, split_range=False): + """ + This will submit a number of timers to the provided connection. It will then ensure that the corresponding + callback is invoked in the appropriate amount of time. + :param unit_test: Invoking unit tests + :param connection: Connection to create the timer on. + :param start: Lower bound of range. + :param end: Upper bound of the time range + :param increment: +1, or -1 + :param precision: 100 for centisecond, 1000 for milliseconds + :param split_range: True to split the range between incrementing and decrementing. + """ + + # Various lists for tracking callback as completed or pending + pending_callbacks = [] + completed_callbacks = [] + + # submit timers with various timeouts + for gross_time in range(start, end, increment): + timeout = get_timeout(gross_time, start, end, precision, split_range) + callback = TimerCallback(timeout) + connection.create_timer(timeout, callback.invoke) + pending_callbacks.append(callback) + + # wait for all the callbacks associated with the timers to be invoked + while len(pending_callbacks) is not 0: + for callback in pending_callbacks: + if callback.was_invoked(): + pending_callbacks.remove(callback) + completed_callbacks.append(callback) + time.sleep(.1) + + # ensure they are all called back in a timely fashion + for callback in completed_callbacks: + unit_test.assertAlmostEqual(callback.expected_wait, callback.get_wait_time(), delta=.1) + + diff --git a/tox.ini b/tox.ini index b6b0b519dc..0b4a0b9f35 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,12 @@ deps = nose PyYAML six + [testenv] deps = {[base]deps} - sure==1.2.3 + sure blist + setenv = USE_CASS_EXTERNAL=1 commands = {envpython} setup.py build_ext --inplace nosetests --verbosity=2 tests/unit/ @@ -19,8 +21,17 @@ commands = {envpython} setup.py build_ext --inplace [testenv:py26] deps = {[testenv]deps} unittest2 + twisted + eventlet + gevent # test skipping is different in unittest2 for python 2.7+; let's just use it where needed +[testenv:py27] +deps = {[testenv]deps} + twisted + eventlet + gevent + [testenv:pypy] deps = {[base]deps} commands = {envpython} setup.py build_ext --inplace From ea27ef120176c7ee13f6a61026e3a949193f7790 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Wed, 15 Jul 2015 16:49:14 -0500 Subject: [PATCH 0277/2431] Adding tests for python-313 --- .../standard/test_custom_protocol_handler.py | 218 ++++++++++++++++++ tox.ini | 2 +- 2 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 tests/integration/standard/test_custom_protocol_handler.py diff --git a/tests/integration/standard/test_custom_protocol_handler.py b/tests/integration/standard/test_custom_protocol_handler.py new file mode 100644 index 0000000000..63d4b1991a --- /dev/null +++ b/tests/integration/standard/test_custom_protocol_handler.py @@ -0,0 +1,218 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + +from cassandra.protocol import ProtocolHandler, ResultMessage, UUIDType, read_int, EventMessage +from cassandra.query import tuple_factory +from cassandra.cluster import Cluster +from tests.integration import use_singledc, PROTOCOL_VERSION, execute_until_pass +from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, get_sample +import uuid + + +def setup_module(): + use_singledc() + update_datatypes() + + +class CustomProtocolHandlerTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.session = cls.cluster.connect() + cls.session.execute("CREATE KEYSPACE custserdes WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}") + cls.session.set_keyspace("custserdes") + + @classmethod + def tearDownClass(cls): + cls.session.execute("DROP KEYSPACE custserdes") + cls.cluster.shutdown() + + def test_custom_raw_uuid_row_results(self): + """ + Test to validate that custom protocol handlers work with raw row results + + Connect and validate that the normal protocol handler is used. + Re-Connect and validate that the custom protocol handler is used. + Re-Connect and validate that the normal protocol handler is used. + + @since 2.7 + @jira_ticket PYTHON-313 + @expected_result custom protocol handler is invoked appropriately. + + @test_category data_types:serialization + """ + + # Ensure that we get normal uuid back first + session = Cluster().connect() + session.row_factory = tuple_factory + result_set = session.execute("SELECT schema_version FROM system.local") + result = result_set.pop() + uuid_type = result[0] + self.assertEqual(type(uuid_type), uuid.UUID) + + # use our custom protocol handlder + + session.client_protocol_handler = CustomTestRawRowType + session.row_factory = tuple_factory + result_set = session.execute("SELECT schema_version FROM system.local") + result = result_set.pop() + raw_value = result.pop() + self.assertEqual(type(raw_value), str) + self.assertEqual(len(raw_value), 16) + + # Ensure that we get normal uuid back when we re-connect + session.client_protocol_handler = ProtocolHandler + result_set = session.execute("SELECT schema_version FROM system.local") + result = result_set.pop() + uuid_type = result[0] + self.assertEqual(type(uuid_type), uuid.UUID) + session.shutdown() + + def test_custom_raw_row_results_all_types(self): + """ + Test to validate that custom protocol handlers work with varying types of + results + + Connect, create a table with all sorts of data. Query the data, make the sure the custom results handler is + used correctly. + + @since 2.7 + @jira_ticket PYTHON-313 + @expected_result custom protocol handler is invoked with various result types + + @test_category data_types:serialization + """ + # Connect using a custom protocol handler that tracks the various types the result message is used with. + session = Cluster().connect(keyspace="custserdes") + session.client_protocol_handler = CustomProtocolHandlerResultMessageTracked + session.row_factory = tuple_factory + + columns_string = create_table_with_all_types("test_table", session) + + # verify data + params = get_all_primitive_params() + results = session.execute("SELECT {0} FROM alltypes WHERE zz=0".format(columns_string))[0] + for expected, actual in zip(params, results): + self.assertEqual(actual, expected) + # Ensure we have covered the various primitive types + self.assertEqual(len(CustomResultMessageTracked.checked_rev_row_set), len(PRIMITIVE_DATATYPES)-1) + session.shutdown() + + +def create_table_with_all_types(table_name, session): + """ + Method that given a table_name and session construct a table that contains all possible primitive types + :param table_name: Name of table to create + :param session: session to use for table creation + :return: a string containing and columns. This can be used to query the table. + """ + # create table + alpha_type_list = ["zz int PRIMARY KEY"] + col_names = ["zz"] + start_index = ord('a') + for i, datatype in enumerate(PRIMITIVE_DATATYPES): + alpha_type_list.append("{0} {1}".format(chr(start_index + i), datatype)) + col_names.append(chr(start_index + i)) + + session.execute("CREATE TABLE alltypes ({0})".format(', '.join(alpha_type_list)), timeout=120) + + # create the input + params = get_all_primitive_params() + + # insert into table as a simple statement + columns_string = ', '.join(col_names) + placeholders = ', '.join(["%s"] * len(col_names)) + session.execute("INSERT INTO alltypes ({0}) VALUES ({1})".format(columns_string, placeholders), params, timeout=120) + return columns_string + + +def get_all_primitive_params(): + """ + Simple utility method used to give back a list of all possible primitive data sample types. + """ + params = [0] + for datatype in PRIMITIVE_DATATYPES: + params.append((get_sample(datatype))) + return params + + +class CustomResultMessageRaw(ResultMessage): + """ + This is a custom Result Message that is used to return raw results, rather then + results which contain objects. + """ + my_type_codes = ResultMessage.type_codes.copy() + my_type_codes[0xc] = UUIDType + type_codes = my_type_codes + + @classmethod + def recv_results_rows(cls, f, protocol_version, user_type_map): + paging_state, column_metadata = cls.recv_results_metadata(f, user_type_map) + rowcount = read_int(f) + rows = [cls.recv_row(f, len(column_metadata)) for _ in range(rowcount)] + coltypes = [c[3] for c in column_metadata] + return (paging_state, (coltypes, rows)) + + +class CustomTestRawRowType(ProtocolHandler): + """ + This is the a custom protocol handler that will substitute the the + customResultMesageRowRaw Result message for our own implementation + """ + my_opcodes = ProtocolHandler.message_types_by_opcode.copy() + my_opcodes[CustomResultMessageRaw.opcode] = CustomResultMessageRaw + message_types_by_opcode = my_opcodes + + +class CustomResultMessageTracked(ResultMessage): + """ + This is a custom Result Message that is use to track what primitive types + have been processed when it receives results + """ + my_type_codes = ResultMessage.type_codes.copy() + my_type_codes[0xc] = UUIDType + type_codes = my_type_codes + checked_rev_row_set = set() + + @classmethod + def recv_results_rows(cls, f, protocol_version, user_type_map): + paging_state, column_metadata = cls.recv_results_metadata(f, user_type_map) + rowcount = read_int(f) + rows = [cls.recv_row(f, len(column_metadata)) for _ in range(rowcount)] + colnames = [c[2] for c in column_metadata] + coltypes = [c[3] for c in column_metadata] + cls.checked_rev_row_set.update(coltypes) + parsed_rows = [ + tuple(ctype.from_binary(val, protocol_version) + for ctype, val in zip(coltypes, row)) + for row in rows] + return (paging_state, (colnames, parsed_rows)) + + +class CustomProtocolHandlerResultMessageTracked(ProtocolHandler): + """ + This is the a custom protocol handler that will substitute the the + CustomTestRawRowTypeTracked Result message for our own implementation + """ + my_opcodes = ProtocolHandler.message_types_by_opcode.copy() + my_opcodes[CustomResultMessageTracked.opcode] = CustomResultMessageTracked + message_types_by_opcode = my_opcodes + + diff --git a/tox.ini b/tox.ini index 23b2aee1f4..b6b0b519dc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py26,py27,pypy,py33,py34 [base] deps = nose - mock + mock<=1.0.1 PyYAML six From 3b7f879514cc472eee56683113e6e29b30c2a0b6 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Fri, 26 Jun 2015 17:25:08 -0500 Subject: [PATCH 0278/2431] Adding Extended SSL tests for client authentication --- tests/integration/long/ssl/driver_ca_cert.pem | 2 +- tests/integration/long/ssl/python_driver.crt | Bin 0 -> 947 bytes tests/integration/long/ssl/python_driver.jks | Bin 0 -> 2306 bytes tests/integration/long/ssl/python_driver.key | 34 ++++ tests/integration/long/ssl/python_driver.pem | 19 +++ .../long/ssl/python_driver_bad.pem | 19 +++ .../long/ssl/python_driver_no_pass.key | 27 ++++ tests/integration/long/ssl/server_cert.pem | 13 ++ .../ssl/{keystore.jks => server_keystore.jks} | Bin tests/integration/long/ssl/server_trust.jks | Bin 0 -> 1677 bytes tests/integration/long/test_ssl.py | 148 ++++++++++++++++-- 11 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 tests/integration/long/ssl/python_driver.crt create mode 100644 tests/integration/long/ssl/python_driver.jks create mode 100644 tests/integration/long/ssl/python_driver.key create mode 100644 tests/integration/long/ssl/python_driver.pem create mode 100644 tests/integration/long/ssl/python_driver_bad.pem create mode 100644 tests/integration/long/ssl/python_driver_no_pass.key create mode 100644 tests/integration/long/ssl/server_cert.pem rename tests/integration/long/ssl/{keystore.jks => server_keystore.jks} (100%) create mode 100644 tests/integration/long/ssl/server_trust.jks diff --git a/tests/integration/long/ssl/driver_ca_cert.pem b/tests/integration/long/ssl/driver_ca_cert.pem index 79ec6af4bb..7e55555767 100644 --- a/tests/integration/long/ssl/driver_ca_cert.pem +++ b/tests/integration/long/ssl/driver_ca_cert.pem @@ -6,7 +6,7 @@ Fw0xNDEwMDMxNTQ2NDdaFw0xNTAxMDExNTQ2NDdaMGoxCzAJBgNVBAYTAlRFMQsw CQYDVQQIEwJDQTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExETAPBgNVBAoTCERhdGFT dGF4MQswCQYDVQQLEwJURTEYMBYGA1UEAxMPUGhpbGlwIFRob21wc29uMIGfMA0G CSqGSIb3DQEBAQUAA4GNADCBiQKBgQChGDwrhpQR0d+NoqilMgsBlR6A2Dd1oMyI -Ue42sU4tN63g5N44Ic4RpTiyaWgnAkP332ok3YAuVbxytwEv2K9HrUSiokAiuinl +Ue42sU4tN63g5N4adasfasdfsWgnAkP332ok3YAuVbxytwEv2K9HrUSiokAiuinl hhHA8CXTHt/1ItzzWj9uJ3Hneb+5lOkXVTZX7Y+q3aSdpx/HnZqn4i27DtLZF0z3 LccWPWRinQIDAQABoyEwHzAdBgNVHQ4EFgQU9WJpUhgGTBBH4xZBCV7Y9YISCp4w DQYJKoZIhvcNAQELBQADgYEAF6e8eVAjoZhfyJ+jW5mB0pXa2vr5b7VFQ45voNnc diff --git a/tests/integration/long/ssl/python_driver.crt b/tests/integration/long/ssl/python_driver.crt new file mode 100644 index 0000000000000000000000000000000000000000..0a419f4eb13cea8866961205155cfc16f91c2f7e GIT binary patch literal 947 zcmXqLVqS01#58>YGZP~d6N|{_iR=cvY@Awc9&O)w85y}*84McR47m+B*_cCF*o2uv zgAIiZ1VJ1QVJ_#yoXoWRqP)yRLlFZZkRZD-cW`1}Nuq*tPGV7_p_qXPNR&&M*Cnwe zF}NhLLcudHSRS(1^Tr{GeQS(aL)5RzJ4QVi3@EX<1{YoH*`YiMd< zW@Kt;ZeU<&5hcNIWC#*4FoX&W9;Y@jDj^3WBP#=Q6C*!^K@%evQxhX2L$AlKqdyl_ z?ArKU;kTS5_^v9{bQsa zIg0U195b~P()hh8G4{ed-E209C*_74en1zuW;g#B~x*&@>IE-e1#k#t7+0tZ&_8Ga4mscYPA}}oi1CfzI zI^nL;(I9teMJ-9|{wpPAX6GA%Rj0mv_I%~HMWRmc&MrDtym9%16Nx1WES#&|Pq#=q z^2Sst3+eMtl|QrGBjgyTFGqFCyeDPH6AOhDH7D)R^D^&vqY|kr6?g1{{fn5mJ#1lW z9O3IHuFH(mds9#?8n6CFoMXDoqu;kHkLFmN-8buO!S~}LabJV3-uGHkW&UQ<)|t}k z2|t$Hf6!Rj`YQWivB7lHvVi?1ceFXnIMYLTgyzh$Z#i-I=aS~bo%c8+CvyFoZyOwN zhGz!%yV8Asm3B_hknVfsU&}kulY61|BDwkX2ltClllXbHW!1qFt;LD@w>O;H)pkea N`V%ji<^;BhNdSf*Rxtno literal 0 HcmV?d00001 diff --git a/tests/integration/long/ssl/python_driver.jks b/tests/integration/long/ssl/python_driver.jks new file mode 100644 index 0000000000000000000000000000000000000000..9a0fd59f738b77c276f06fc0048a0b52fe64b345 GIT binary patch literal 2306 zcmd6o`8U*$8pmhMUl2=ee@dB9Mx2A2pJ%ngFFpqupzncJdI1r=?# z&f#Ba%M?93!9aIFimfzz#$~&b5}AdIf2O7QGp%hC^k#y1A4Qbw*{J?g7gOx)l71$* zGhtkyj3TVmmwt&|+sapPQ18)Fm*Z6)T^K)Fp);#BN?i_WL_B)z@vY|FgPNDT8iPG_ zYgdx=r?7$Wj^Ia^rh=bxrK(9dpYQ zOjPrdN}sAztJp^0m|Dr0q{xEQaG1c@F|&s}oQNjn{eLkSS1eP;l*eTtM(*fFc$mD2 zTd_N?vb1jIC=WkpWW^5#d#J2UAg>RA%;njPiEB}V8uvb2^ze!ec<>_TD^)b8*OC@+ zitTSRyx88_a3C4y2;TS>d(PV=kBPNWH5w7On&~!j&YZWJd|Z&*WGX1DQe`w+Mrd|0 zTMmJt1;`ksdv5#DYiLidK#LEUi;uIe(ftZ`HU5OKqYC1y1GG072czM^`TXl!8-69z zm|F&ZhFw}i34H{i$#55kI*H!XMRrWL@Vv=rPy5Sie{5-um(w`Lp^7~%G3FTNE=aKF z4ejLlv^(t*_!94cCl-Wva-5n=WqNgPH`@0aDM4MCs@(fW+~y+sJw!%rKNn9~&6Sbt%-y~&_KQfWu8+ww; zJ1pZg*HiwmlU<4NlpQicP@YR-$G%%=<(e%v!_)%=&&LvLQeLHdJpb56aQ>K#r6kU) zB#+{Ib&!N))e3`{x$LPU+s_&gI*1L?{KA@$eu4g^smcK+X&{Fh6hnjCvQnm=I2jF< zySC=T2p>5GtO(#>I|ldqXGR+0mA zHTeT(osl5jkhZq1=+|P~W%`!_%$GQZ&t6fQu1$;U(QieVz-&AJaV+1i!A*rMVJ6^u zP>Y@D=zCiivC{)qBU!a)8jY(9rm>@B@MXgy6rjv?Ij^T0z$S3k?tU8hy{KlH$C z&^nv2rY~yauJ1#$U+BAbyLV)h%l0f4Mv%4#wx1`o2_GL(IG*e=i=ChFNx)mjKz%(I z)kIQSjC$+Xk@ZET+UC06+7%&*LUQ@+)jeMa>Ch3xt?-+x+Ny&y9M8$(^Z1gm@xYjY zt0hCq0`aR~)LX%hr85OjY|deR+mTmk#%Vfx?@$GIw~%(?XYJ zMO`#HpOV^L$;_L*Z5%hMlbkHH=aG+^%Q(p*M7qXDcax&7izeU2AEdfA&q&e(vtI0a z+_$`osoRLcTz)Baa_x?T%1>gptc%@pFex?92mP6_Qeo!&ass`PyjHDu;Wl9wcKvrg za=U5a0RjjFX#!Y~2LOxfMj3<)%*6#o?dI|V2p+gbidD*21Q^WE4FVEVaQpzA2jUFn z5#@4r!ifSRzZ#e*pD8sY&_66PG?0oz0mxsFSCrq08cL_in}$#$sW@=}^^5X}BFw0C zsuP_WBY!&doEA0sD@$kt&SYRO#zykAevA|$ZmKA&OTSW}J@vHp4EE>Eu z%!zAmpB2#$tctYO>76O{H{2aH*hP;|YNqO=@{I-N z;Y|X{<-jtt8I@rvoS*%ajro+d%byfyKX2z_7-%&wCY;O;cijZT8FnYQ6XE1>m8Lg; zH34T`sFr;jFI&{9`|s*^EKZI>VLjbK(nA*?Zx-Ba+Z?b_wHB%`z_GHxtLoa0C~?DF zzw)^TpEEhRy|VjJ3}tBi)LSy8lgC9B=K3JFK9HjIKAb6bR_(n6?1uEp{ybwa ssH7g7Zx~^xEGW;eSxaB`=E?PGO<$;UD<*~oO=dDeTg@A{mK0L{4ZtwgI{*Lx literal 0 HcmV?d00001 diff --git a/tests/integration/long/ssl/python_driver.key b/tests/integration/long/ssl/python_driver.key new file mode 100644 index 0000000000..afd73b298c --- /dev/null +++ b/tests/integration/long/ssl/python_driver.key @@ -0,0 +1,34 @@ +Bag Attributes + friendlyName: python_driver + localKeyID: 54 69 6D 65 20 31 34 33 35 33 33 33 34 30 34 33 33 32 +Key Attributes: +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,8A0BC9CFBBB36D47 + +J3Rh82LhsNdIdCV4KCp758VIJJnmedwtq/I9oxH5kY4XoUQjfNcvLGlEnbAUD6+N +mYnQ5XPDvD7iC19XvlA9gfaoWERq+zroGEP+e4dX1X5RlT6YQBJpJR8IW4DWngDM +Nv6CuaGFJWMH8QUvKlJyFOPOHBqbhsCRaxg3pOG3RyUFXpGPDV0ySUyp6poHE9KE +pEVif/SdS3AhV2sb4tyBS9sRZdH1eeCN4gY6k9PQWyNViAgUYAG5xWsE4fITa3qY +gisyzbOYU8ue2QvmjPJgieiKPQf+st/ZRV5eQUCdUgAfLEnULGJXRZ5kw7kMXL0X +gLaKFbGxv4pKQCDCZQq4GXIA/nmTy6cme+VPKwq3usm+GdxfdWQJjgG65+AFaut/ +XjGm1fvSQzWuzpesfLy57HMK+bBh1/UKjuQa3wAHtgPtJLtUSW+/qBnQRdBbl20C +dJtJXyyTlX6H8bQBIfBLc4ntUwS8fVd2jsYJRpCBY6HdtpfsZZ5gQnm1Nux4ksKn +rYYx3I/JpChr7AV7Yj/lwc3Zca/VJl16CjyWeRTQEvkl6GK958aIzj73HfXleZc6 +HGVfOgo2BLmOzY0ZCq/Wa4fnURRgrC3SusrT9mjVbID91oNYw4BjMEU53u0uxPC+ +rr6SwG2EUVawGTVK4XZw2DINCPP/wsKqf0xqA+sxArcTN/MEdLUBdf8oDntkj2jG +Oy0kwpjqhSvWo1DqYKZjV/wKT2SS18OMAW+0qplbHw1/FDGWK+OseD8MXwBo06a5 +LWRQXhf0kEXUQ+oNj3eahe/npHiNChR6mEiIbCuE3NAXPPXJNkhMuj2f5EqrOPfZ +jqbNiLfKKx7L5t6B8LXkdKGPqztcFlnB8rRF9Eqa8F4wiEg8MBLrPyxgd/uT+NIz +LdDgvUE+IkCwQoYoCU70ApiEOyQNacuSxwUiVWVyn9CJYXPM4Vlje7GDIDRR5Xp6 +zNf0ktNP46PsRqDlYG9hZWndj4PRaAqtatlEEm37rmyouVBe3rxcbL1b1zsH/p1I +eaGGTyZ8+iEiuEk4gCOmfmYmpE7H/DXlQvtDRblid/bEY64Uietx0HQ5yZwXZYi8 +hb4itke6xkgRQEIXVyQOdU88PEuA5yofEGoXkfdLgtdu3erPrVDc+nQTYrMWNacR +JQljfhAFJdjOw81Yd5PnFHAtxcxzqEkWv0TGQLL1VjJdinhI7q/fIPLJ76FtuGmt +zlxo/Jy1aaUgM/e485+7aoNSGi2/t6zGqGuotdUCO5epgrUHX+9fOJnnrYTG9ixp +FSHTT69y72khnw3eMP8NnOS3Lu+xLEzQHNbUDfB8uyVEX4pyA3FPVVqwIaeJDiPS +2x7Sl5KKwLbqPPKRFRC1qLsN4KcqeXBG+piTLPExdzsLbrU9JZMcaNmSmUabdg20 +SCwIuU2kHEpO7O7yNGeV9m0CGFUaoCAHVG70oXHxpVjAJbtgyoBkiwSxghCxXkfW +Mg+1B2k4Gk1WrLjIyasH6p0MLUJ7qLYN+c+wF7ms00F/w04rM6zUpkgnqsazpw6F +weUhpA8qY2vOJN6rsB4byaOUnd33xhAwcY/pIAcjW7UBjNmFMB1DQg== +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/long/ssl/python_driver.pem b/tests/integration/long/ssl/python_driver.pem new file mode 100644 index 0000000000..83556fd9ce --- /dev/null +++ b/tests/integration/long/ssl/python_driver.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIEFPORBzANBgkqhkiG9w0BAQsFADCBhjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMRYwFAYDVQQKEw1EYXRhU3RheCBJ +bmMuMRwwGgYDVQQLExNQeXRob24gRHJpdmVyIFRlc3RzMRYwFAYDVQQDEw1QeXRob24gRHJpdmVy +MCAXDTE1MDYyNTE3MDAxOFoYDzIxMTUwNjAxMTcwMDE4WjCBhjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMRYwFAYDVQQKEw1EYXRhU3RheCBJ +bmMuMRwwGgYDVQQLExNQeXRob24gRHJpdmVyIFRlc3RzMRYwFAYDVQQDEw1QeXRob24gRHJpdmVy +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAjUi6xfmieLqx9yD7HhkB6sjHfbS51xE7 +aaRySjTA1p9mPfPMPPMZ0NIsiDsUlT7Fa+LWZU9cGuJBFg8YxjU5Eij7smFd0J4tawk51KudDdUZ +crALGFC3WY7sEboMl8UHqV+kESPlNm5/JSNSYkNm1TMi9mHcB/Bg3RDpORRW/keMtBSLRxCVjsu6 +GvKN8wuEfU/bTmI9aUjbFRCFunBX6QEJeU44BYEJXNAls+X8szBfVmFHwefatSlh++uu7kY6zAQI +v74PHMZ8w+mWmbjpxEsmSg+uljGCjQHjKTNSFBY9kWWh2LBiTcZuEsQ9DK0J/+1tUa0s5vq6CjUK +XRxwpQIDAQABoyEwHzAdBgNVHQ4EFgQUJwTYG8dcZDt7faalYwCHmG3jp3swDQYJKoZIhvcNAQEL +BQADggEBABtg3SLFUkcbISoZO4/UdHY2z4BTJZXt5uep9qIVQu7NospzsafgyGF0YAQJq0fLhBlB +DVx6IxIvDZUfzKdIVMYJTQh7ZJ7kdsdhcRIhKZK4Lko3iOwkWS0aXsbQP+hcXrwGViYIV6+Rrmle +LuxwexVfJ+wXCJcc4vvbecVsOs2+ms1w98cUXvVS1d9KpHo37LK1mRsnYPik3+CBeYXqa8FzMJc1 +dlC/dNwrCXYJZ1QMEpyaP4TI3fmkg8OJ3glZkQr6nz1TUMwMmAvudb79IrmQKBuO6k99DZFJC6Er +oh6ff8G/F5YY+dWEqsF0KqNhL9uwyrqG3CTX5Eocg2AGkWI= +-----END CERTIFICATE----- diff --git a/tests/integration/long/ssl/python_driver_bad.pem b/tests/integration/long/ssl/python_driver_bad.pem new file mode 100644 index 0000000000..978d6c53f3 --- /dev/null +++ b/tests/integration/long/ssl/python_driver_bad.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIEFPORBzANBgkqhkiG9w0BAQsFADCBhjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMRYwFAYDVQQKEw1EYXRhU3RheCBJ +bmMuMRwwGgYDVQQLExNQeXRob24gRHJpdmVyIFRlc3RzMRYwFAYDVQQDEw1QeXRob24gRHJpdmVy +MCAXDTE1MDYyNTE3MDAxOFoYDzIxMTUwNjAxMTcwMDE4WjCBhjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMRYwFAYDVQQKEw1EYXRhU3RheCBJ +bmMuMRwwGgYDVQQLExNQeXRob24gRHJpdmVyIFRlc3RzMRYwFAYDVQQDEw1QeXRob24gRHJpdmVy +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAjUi6xfmieLqx9yD7HhkB6sjHfbS51xE7 +aaRySjTA1p9mPfPMPPMZ0NIsiDsUlT7Fa+LWZU9cGuJBFg8YxjU5Eij7smFd0J4tawk51KudDdUZ +crALGFC3WY7sEboMl8UHqV+kESPlNm5/JSNSYkNm1TMi9mHcB/Bg3RDpORRW/keMtBSLRxCVjsu6 +GvKN8wuEfU/bTmI9aUjbFRCFunBX6QEJeU44BYEJXNAls+X8szBfVmFHwefatSlh++uu7kY6zAQI +v74PHMZ8w+mWmbjpxEsmSg+uljGCjQHjKTNSFBY9kWWh2LBiTcZuEsQ9DK0J/+1tUa0s5vq6CjUK +XRxwpQIDAQABoyE666666gNVHQ4EFgQUJwTYG8dcZDt7faalYwCHmG3jp3swDQYJKoZIhvcNAQEL +BQADggEBABtg3SLFUkcbISoZO4/UdHY2z4BTJZXt5uep9qIVQu7NospzsafgyGF0YAQJq0fLhBlB +DVx6IxIvDZUfzKdIVMYJTQh7ZJ7kdsdhcRIhKZK4Lko3iOwkWS0aXsbQP+hcXrwGViYIV6+Rrmle +LuxwexVfJ+wXCJcc4vvbecVsOs2+ms1w98cUXvVS1d9KpHo37LK1mRsnYPik3+CBeYXqa8FzMJc1 +dlC/dNwrCXYJZ1QMEpyaP4TI3fmkg8OJ3glZkQr6nz1TUMwMmAvudb79IrmQKBuO6k99DZFJC6Er +oh6ff8G/F5YY+dWEqsF0KqNhL9uwyrqG3CTX5Eocg2AGkWI= +-----END CERTIFICATE----- diff --git a/tests/integration/long/ssl/python_driver_no_pass.key b/tests/integration/long/ssl/python_driver_no_pass.key new file mode 100644 index 0000000000..8dd14f84f0 --- /dev/null +++ b/tests/integration/long/ssl/python_driver_no_pass.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAjUi6xfmieLqx9yD7HhkB6sjHfbS51xE7aaRySjTA1p9mPfPM +PPMZ0NIsiDsUlT7Fa+LWZU9cGuJBFg8YxjU5Eij7smFd0J4tawk51KudDdUZcrAL +GFC3WY7sEboMl8UHqV+kESPlNm5/JSNSYkNm1TMi9mHcB/Bg3RDpORRW/keMtBSL +RxCVjsu6GvKN8wuEfU/bTmI9aUjbFRCFunBX6QEJeU44BYEJXNAls+X8szBfVmFH +wefatSlh++uu7kY6zAQIv74PHMZ8w+mWmbjpxEsmSg+uljGCjQHjKTNSFBY9kWWh +2LBiTcZuEsQ9DK0J/+1tUa0s5vq6CjUKXRxwpQIDAQABAoIBAC3bpYQM+wdk0c79 +DYU/aLfkY5wRxSBhn38yuUYMyWrgYjdJoslFvuNg1MODKbMnpLzX6+8GS0cOmUGn +tMrhC50xYEEOCX1lWiib3gGBkoCi4pevPGqwCFMxaL54PQ4mDc6UFJTbqdJ5Gxva +0yrB5ebdqkN+kASjqU0X6Bt21qXB6BvwAgpIXSX8r+NoH2Z9dumSYD+bOwhXo+/b +FQ1wyLL78tDdlJ8KibwnTv9RtLQbALUinMEHyP+4Gp/t/JnxlcAfvEwggYBxFR1K +5sN8dMFbMZVNqNREXZyWCMQqPbKLhIHPHlNo5pJP7cUh9iVH4QwYNIbOqUza/aUx +z7DIISECgYEAvpAAdDiBExMOELz4+ku5Uk6wmVOMnAK6El4ijOXjJsOB4FB6M0A6 +THXlzLws0YLcoZ3Pm91z20rqmkv1VG+En27uKC1Dgqqd4DOQzMuPoPxzq/q2ozFH +V5U1a0tTmyynr3CFzQUJKLJs1pKKIp6HMiB48JWQc5q6ZaaomEnOiYsCgYEAvczB +Bwwf7oaZGhson1HdcYs5kUm9VkL/25dELUt6uq5AB5jjvfOYd7HatngNRCabUCgE +gcaNfJSwpbOEZ00AxKVSxGmyIP1YAlkVcSdfAPwGO6C1+V4EPHqYUW0AVHOYo7oB +0MCyLT6nSUNiHWyI7qSEwCP03SqyAKA1pDRUVI8CgYBt+bEpYYqsNW0Cn+yYlqcH +Jz6n3h3h03kLLKSH6AwlzOLhT9CWT1TV15ydgWPkLb+ize6Ip087mYq3LWsSJaHG +WUC8kxLJECo4v8mrRzdG0yr2b6SDnebsVsITf89qWGUVzLyLS4Kzp/VECCIMRK0F +ctQZFFffP8ae74WRDddSbQKBgQC7vZ9qEyo6zNUAp8Ck51t+BtNozWIFw7xGP/hm +PXUm11nqqecMa7pzG3BWcaXdtbqHrS3YGMi3ZHTfUxUzAU4zNb0LH+ndC/xURj4Z +cXJeDO01aiDWi5LxJ+snEAT1hGqF+WX2UcVtT741j/urU0KXnBDb5jU92A++4rps +tH5+LQKBgGHtOWD+ffKNw7IrVLhP16GmYoZZ05zh10d1eUa0ifgczjdAsuEH5/Aq +zK7MsDyPcQBH/pOwAcifWGEdXmn9hL6w5dn96ABfa8Qh9nXWrCE2OFD81PDU9Osd +wnwbTKlYWPBwdF7UCseKC7gXkUD6Ls0ADWJvrCI7AfQJv6jj6nnE +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/long/ssl/server_cert.pem b/tests/integration/long/ssl/server_cert.pem new file mode 100644 index 0000000000..7c96b96ade --- /dev/null +++ b/tests/integration/long/ssl/server_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICbjCCAdegAwIBAgIEP/N06DANBgkqhkiG9w0BAQsFADBqMQswCQYDVQQGEwJURTELMAkGA1UE +CBMCQ0ExFDASBgNVBAcTC1NhbnRhIENsYXJhMREwDwYDVQQKEwhEYXRhU3RheDELMAkGA1UECxMC +VEUxGDAWBgNVBAMTD1BoaWxpcCBUaG9tcHNvbjAeFw0xNDEwMDMxNTQ2NDdaFw0xNTAxMDExNTQ2 +NDdaMGoxCzAJBgNVBAYTAlRFMQswCQYDVQQIEwJDQTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExETAP +BgNVBAoTCERhdGFTdGF4MQswCQYDVQQLEwJURTEYMBYGA1UEAxMPUGhpbGlwIFRob21wc29uMIGf +MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQChGDwrhpQR0d+NoqilMgsBlR6A2Dd1oMyIUe42sU4t +N63g5N44Ic4RpTiyaWgnAkP332ok3YAuVbxytwEv2K9HrUSiokAiuinlhhHA8CXTHt/1ItzzWj9u +J3Hneb+5lOkXVTZX7Y+q3aSdpx/HnZqn4i27DtLZF0z3LccWPWRinQIDAQABoyEwHzAdBgNVHQ4E +FgQU9WJpUhgGTBBH4xZBCV7Y9YISCp4wDQYJKoZIhvcNAQELBQADgYEAF6e8eVAjoZhfyJ+jW5mB +0pXa2vr5b7VFQ45voNncGrB3aNbz/AWT7LCJw88+Y5SJITgwN/8o3ZY6Y3MyiqeQYGo9WxDSWb5A +dZWFa03Z+hrVDQuw1r118zIhdS4KYDCQM2JfWY32TwK0MNG/6BO876HfkDpcjCYzq8Gh0gEguOA= +-----END CERTIFICATE----- diff --git a/tests/integration/long/ssl/keystore.jks b/tests/integration/long/ssl/server_keystore.jks similarity index 100% rename from tests/integration/long/ssl/keystore.jks rename to tests/integration/long/ssl/server_keystore.jks diff --git a/tests/integration/long/ssl/server_trust.jks b/tests/integration/long/ssl/server_trust.jks new file mode 100644 index 0000000000000000000000000000000000000000..feb0784a06189e4abb5d763d0bf011590a311c1f GIT binary patch literal 1677 zcmezO_TO6u1_mZL=1fj3E>6r#DN1BuVD!`BoBN-EHA2tSz!Ipa$e@WS&!CC%`T}Mq zMkXc}`_CmW40zc%wc0$|zVk9Nayplu(=bXf%L_X)h&(lkwb!!m4Ugjm%*U1lc}+h zVWEVLcH0!ei}!mMtypTr%{W!A;f8tXf-@a~@60y(>6)*7@Z_F_;yJ;k7Mn6N)R~;W z-_KIH+n^V^r)WE){*CqSYh4yCa!}f(`Ls>&zz5aKa`(R~-T544pQm2P(0Ta0UGkJpMGFJ-{~C9vStS=6buFKekYyV!a4B-1 zL+RAkY~P!|q^|OEZ@9L%^s|v-sUBB?!35)^_{iRG{!Cj8F7AIJyyyMG`xC5Uden?p zA6$5eQDMgeaJu9zs4U6I&x=nf$}CGQ0;fxU^FDaGWZrDh#JnDuHm5_=rO4-r>}Y;5 zXl#R}DqzA44K@@u5Co+v4q-0m#GK5u{GzviJ%s>Q^KzU&aRKYVZS=`RIzL0cZJ__l8moT9IxH7^SYpQ=8_^W zlLOc0r`di!WAj<^!X=#!YmupTN3$PYOZAVDdgLg^FLBJ&Qb^2X_BpmkS4_)A94No^pnS!O*@|8Nv4e8@CDc04s%a$fHw9m+W zyu2DY5JCA06n4@Hca@F?xl1c*Nm}<`DJe5M-w>=i_3g9gE59ugb$WMp(W&B%%O9Lb zEJ&=L*$0aarkj=p>@T^a%~{5o9>OCuXO?}-iMv0SG#~D~ z#~C@1>(_kS;D9qcGq~TC?)$5>bApC+-z)!G-ie;v3$+)?&96VWUwoRx&#Nt~4wh&w tPSn4>;nc3SJ1W 5: + raise RuntimeError("Failed to connect to SSL cluster after 5 attempts") + try: + cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, + 'ssl_version': ssl.PROTOCOL_TLSv1, + 'keyfile': abs_driver_keyfile, + 'certfile': abs_driver_certfile}) + + session = cluster.connect() + break + except Exception: + ex_type, ex, tb = sys.exc_info() + log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) + del tb + tries += 1 + + # attempt a few simple commands. + + insert_keyspace = """CREATE KEYSPACE ssltest + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} + """ + statement = SimpleStatement(insert_keyspace) + statement.consistency_level = 3 + session.execute(statement) + + drop_keyspace = "DROP KEYSPACE ssltest" + statement = SimpleStatement(drop_keyspace) + statement.consistency_level = ConsistencyLevel.ANY + session.execute(statement) + + cluster.shutdown() + + def test_cannot_connect_without_client_auth(self): + """ + Test to validate that we cannot connect without client auth. + + This test will omit the keys/certs needed to preform client authentication. It will then attempt to connect + to a server that has client authentication enabled. + + @since 2.7.0 + @expected_result The client will throw an exception on connect + + @test_category connection:ssl + """ + + abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS) + cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, + 'ssl_version': ssl.PROTOCOL_TLSv1}) + # attempt to connect and expect an exception + + with self.assertRaises(NoHostAvailable) as context: + cluster.connect() + + def test_cannot_connect_with_bad_client_auth(self): + """ + Test to validate that we cannot connect with invalid client auth. + + This test will use bad keys/certs to preform client authentication. It will then attempt to connect + to a server that has client authentication enabled. + + + @since 2.7.0 + @expected_result The client will throw an exception on connect + + @test_category connection:ssl + """ + + # Setup absolute paths to key/cert files + abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS) + abs_driver_keyfile = os.path.abspath(DRIVER_KEYFILE) + abs_driver_certfile = os.path.abspath(DRIVER_CERTFILE_BAD) + + cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, + 'ssl_version': ssl.PROTOCOL_TLSv1, + 'keyfile': abs_driver_keyfile, + 'certfile': abs_driver_certfile}) + with self.assertRaises(NoHostAvailable) as context: + cluster.connect() \ No newline at end of file From 04bbff3d3b5666636de94edb9c33549bb0043af6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 21 Jul 2015 14:54:24 -0500 Subject: [PATCH 0279/2431] Unlock while yielding execute concurrent generator results Fixes an issue where the event thread could be held up on the executor lock while the client thread waits for a paged result to return. --- cassandra/concurrent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index aba3045605..7c0d3fd7e1 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -166,9 +166,11 @@ def _results(self): self._condition.wait() while self._results_queue and self._results_queue[0][0] == self._current: _, res = heappop(self._results_queue) + self._condition.release() if self._fail_fast and not res[0]: self._raise(res[1]) yield res + self._condition.acquire() self._current += 1 From 791f44c6ddd7eac44013e887e38c6f21de4d5dd7 Mon Sep 17 00:00:00 2001 From: GregBestland Date: Mon, 20 Jul 2015 14:19:47 -0500 Subject: [PATCH 0280/2431] Adding unit and integration tests for PYTHON-123. --- tests/integration/standard/test_concurrent.py | 87 ++++++- tests/unit/test_concurrent.py | 227 ++++++++++++++++++ 2 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_concurrent.py diff --git a/tests/integration/standard/test_concurrent.py b/tests/integration/standard/test_concurrent.py index 9a82338348..45b736132c 100644 --- a/tests/integration/standard/test_concurrent.py +++ b/tests/integration/standard/test_concurrent.py @@ -50,11 +50,11 @@ def setUpClass(cls): def tearDownClass(cls): cls.cluster.shutdown() - def execute_concurrent_helper(self, session, query): + def execute_concurrent_helper(self, session, query, results_generator=False): count = 0 while count < 100: try: - return execute_concurrent(session, query) + return execute_concurrent(session, query, results_generator=False) except (ReadTimeout, WriteTimeout, OperationTimedOut, ReadFailure, WriteFailure): ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) @@ -63,11 +63,11 @@ def execute_concurrent_helper(self, session, query): raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(query)) - def execute_concurrent_args_helper(self, session, query, params): + def execute_concurrent_args_helper(self, session, query, params, results_generator=False): count = 0 while count < 100: try: - return execute_concurrent_with_args(session, query, params) + return execute_concurrent_with_args(session, query, params, results_generator=results_generator) except (ReadTimeout, WriteTimeout, OperationTimedOut, ReadFailure, WriteFailure): ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) @@ -120,6 +120,40 @@ def test_execute_concurrent_with_args(self): self.assertEqual(num_statements, len(results)) self.assertEqual([(True, [(i,)]) for i in range(num_statements)], results) + def test_execute_concurrent_with_args_generator(self): + """ + Test to validate that generator based results are surfaced correctly + + Repeatedly inserts data into a a table and attempts to query it. It then validates that the + results are returned in the order expected + + @since 2.7.0 + @jira_ticket PYTHON-123 + @expected_result all data should be returned in order. + + @test_category queries:async + """ + for num_statements in (0, 1, 2, 7, 10, 99, 100, 101, 199, 200, 201): + statement = SimpleStatement( + "INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", + consistency_level=ConsistencyLevel.QUORUM) + parameters = [(i, i) for i in range(num_statements)] + + results = self.execute_concurrent_args_helper(self.session, statement, parameters, results_generator=True) + for result in results: + self.assertEqual((True, None), result) + + # read + statement = SimpleStatement( + "SELECT v FROM test3rf.test WHERE k=%s", + consistency_level=ConsistencyLevel.QUORUM) + parameters = [(i, ) for i in range(num_statements)] + + results = self.execute_concurrent_args_helper(self.session, statement, parameters, results_generator=True) + for i in range(num_statements): + result = results.next() + self.assertEqual((True, [(i,)]), result) + def test_execute_concurrent_paged_result(self): if PROTOCOL_VERSION < 2: raise unittest.SkipTest( @@ -150,6 +184,51 @@ def test_execute_concurrent_paged_result(self): self.assertIsInstance(result, PagedResult) self.assertEqual(num_statements, sum(1 for _ in result)) + def test_execute_concurrent_paged_result_generator(self): + """ + Test to validate that generator based results are surfaced correctly when paging is used + + Inserts data into a a table and attempts to query it. It then validates that the + results are returned as expected (no order specified) + + @since 2.7.0 + @jira_ticket PYTHON-123 + @expected_result all data should be returned in order. + + @test_category paging + """ + if PROTOCOL_VERSION < 2: + raise unittest.SkipTest( + "Protocol 2+ is required for Paging, currently testing against %r" + % (PROTOCOL_VERSION,)) + + num_statements = 201 + statement = SimpleStatement( + "INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", + consistency_level=ConsistencyLevel.QUORUM) + parameters = [(i, i) for i in range(num_statements)] + + results = self.execute_concurrent_args_helper(self.session, statement, parameters, results_generator=True) + self.assertEqual(num_statements, sum(1 for _ in results)) + + # read + statement = SimpleStatement( + "SELECT * FROM test3rf.test LIMIT %s", + consistency_level=ConsistencyLevel.QUORUM, + fetch_size=int(num_statements / 2)) + parameters = [(i, ) for i in range(num_statements)] + + paged_results_gen = self.execute_concurrent_args_helper(self.session, statement, [(num_statements,)], results_generator=True) + + # iterate over all the result and make sure we find the correct number. + found_results = 0 + for result_tuple in paged_results_gen: + paged_result = result_tuple[1] + for _ in paged_result: + found_results += 1 + + self.assertEqual(found_results, num_statements) + def test_first_failure(self): statements = cycle(("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", )) parameters = [(i, i) for i in range(100)] diff --git a/tests/unit/test_concurrent.py b/tests/unit/test_concurrent.py new file mode 100644 index 0000000000..5e23f584a0 --- /dev/null +++ b/tests/unit/test_concurrent.py @@ -0,0 +1,227 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa +from itertools import cycle +from mock import Mock +import time +import threading +from six.moves.queue import PriorityQueue + +from cassandra.concurrent import execute_concurrent, execute_concurrent_with_args + + +class MockResponseResponseFuture(): + """ + This is a mock ResponseFuture. It is used to allow us to hook into the underlying session + and invoke callback with various timing. + """ + + # a list pending callbacks, these will be prioritized in reverse or normal orderd + pending_callbacks = PriorityQueue() + + def __init__(self, reverse): + + # if this is true invoke callback in the reverse order then what they were insert + self.reverse = reverse + # hardcoded to avoid paging logic + self.has_more_pages = False + + if(reverse): + self.priority = 100 + else: + self.priority = 0 + + def add_callback(self, fn, *args, **kwargs): + """ + This is used to add a callback our pending list of callbacks. + If reverse is specified we will invoke the callback in the opposite order that we added it + """ + time_added = time.time() + self.pending_callbacks.put((self.priority, (fn, args, kwargs, time_added))) + if not reversed: + self.priority += 1 + else: + self.priority -= 1 + + def add_callbacks(self, callback, errback, + callback_args=(), callback_kwargs=None, + errback_args=(), errback_kwargs=None): + + self.add_callback(callback, *callback_args, **(callback_kwargs or {})) + + def get_next_callback(self): + return self.pending_callbacks.get() + + def has_next_callback(self): + return not self.pending_callbacks.empty() + + def has_more_pages(self): + return False + + def clear_callbacks(self): + return + + +class TimedCallableInvoker(threading.Thread): + """ + This is a local thread which is runs and invokes all the callbacks on the pending callback queue. + The slowdown flag can used to invoke random slowdowns in our simulate queries. + """ + def __init__(self, handler, slowdown=False): + super(TimedCallableInvoker, self).__init__() + self.slowdown = slowdown + self._stop = threading.Event() + self.handler = handler + + def stop(self): + self._stop.set() + + def stopped(self): + return self._stop.isSet() + + def run(self): + while(not self.stopped()): + if(self.handler.has_next_callback()): + pending_callback = self.handler.get_next_callback() + priority_num = pending_callback[0] + if (priority_num % 10) == 0 and self.slowdown: + self._stop.wait(.1) + callback_args = pending_callback[1] + fn, args, kwargs, time_added = callback_args + fn(time_added, *args, **kwargs) + self._stop.wait(.001) + return + + +class ConcurrencyTest((unittest.TestCase)): + + def test_results_ordering_forward(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorListResults + when queries complete in the order they were executed. + """ + self.insert_and_validate_list_results(False, False) + + def test_results_ordering_reverse(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorListResults + when queries complete in the reverse order they were executed. + """ + self.insert_and_validate_list_results(True, False) + + def test_results_ordering_forward_slowdown(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorListResults + when queries complete in the order they were executed, with slow queries mixed in. + """ + self.insert_and_validate_list_results(False, True) + + def test_results_ordering_reverse_slowdown(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorListResults + when queries complete in the reverse order they were executed, with slow queries mixed in. + """ + self.insert_and_validate_list_results(True, True) + + def test_results_ordering_forward_generator(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorGenResults + when queries complete in the order they were executed. + """ + self.insert_and_validate_list_generator(False, False) + + def test_results_ordering_reverse_generator(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorGenResults + when queries complete in the reverse order they were executed. + """ + self.insert_and_validate_list_generator(True, False) + + def test_results_ordering_forward_generator_slowdown(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorGenResults + when queries complete in the order they were executed, with slow queries mixed in. + """ + self.insert_and_validate_list_generator(False, True) + + def test_results_ordering_reverse_generator_slowdown(self): + """ + This tests the ordering of our various concurrent generator class ConcurrentExecutorGenResults + when queries complete in the reverse order they were executed, with slow queries mixed in. + """ + self.insert_and_validate_list_generator(True, True) + + def insert_and_validate_list_results(self, reverse, slowdown): + """ + This utility method will execute submit various statements for execution using the ConcurrentExecutorListResults, + then invoke a separate thread to execute the callback associated with the futures registered + for those statements. The parameters will toggle various timing, and ordering changes. + Finally it will validate that the results were returned in the order they were submitted + :param reverse: Execute the callbacks in the opposite order that they were submitted + :param slowdown: Cause intermittent queries to perform slowly + """ + our_handler = MockResponseResponseFuture(reverse=reverse) + mock_session = Mock() + statements_and_params = zip(cycle(["INSERT INTO test3rf.test (k, v) VALUES (%s, 0)"]), + [(i, ) for i in range(100)]) + mock_session.execute_async.return_value = our_handler + + t = TimedCallableInvoker(our_handler, slowdown=slowdown) + t.start() + results = execute_concurrent(mock_session, statements_and_params) + + while(not our_handler.pending_callbacks.empty()): + time.sleep(.01) + t.stop() + self.validate_result_ordering(results) + + def insert_and_validate_list_generator(self, reverse, slowdown): + """ + This utility method will execute submit various statements for execution using the ConcurrentExecutorGenResults, + then invoke a separate thread to execute the callback associated with the futures registered + for those statements. The parameters will toggle various timing, and ordering changes. + Finally it will validate that the results were returned in the order they were submitted + :param reverse: Execute the callbacks in the opposite order that they were submitted + :param slowdown: Cause intermittent queries to perform slowly + """ + our_handler = MockResponseResponseFuture(reverse=reverse) + mock_session = Mock() + statements_and_params = zip(cycle(["INSERT INTO test3rf.test (k, v) VALUES (%s, 0)"]), + [(i, ) for i in range(100)]) + mock_session.execute_async.return_value = our_handler + + t = TimedCallableInvoker(our_handler, slowdown=slowdown) + t.start() + results = execute_concurrent(mock_session, statements_and_params, results_generator=True) + + self.validate_result_ordering(results) + t.stop() + + def validate_result_ordering(self, results): + """ + This method will validate that the timestamps returned from the result are in order. This indicates that the + results were returned in the order they were submitted for execution + :param results: + """ + last_time_added = 0 + for result in results: + current_time_added = result[1] + self.assertLess(last_time_added, current_time_added) + last_time_added = current_time_added + From 17f2bc13c2fda731e9de8217790730b151f962ec Mon Sep 17 00:00:00 2001 From: Kevin Deldycke Date: Wed, 22 Jul 2015 15:10:54 +0200 Subject: [PATCH 0281/2431] Unit-tests CQLengine's previous value tracking. Test previous_value of an already-persisted object as well as one that was just instanciated. Refs PYTHON-348. --- .../cqlengine/model/test_model_io.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/integration/cqlengine/model/test_model_io.py b/tests/integration/cqlengine/model/test_model_io.py index 3ba2aa9711..b6f76599e7 100644 --- a/tests/integration/cqlengine/model/test_model_io.py +++ b/tests/integration/cqlengine/model/test_model_io.py @@ -347,6 +347,80 @@ def test_get_changed_columns(self): self.instance.save() assert self.instance.get_changed_columns() == [] + def test_previous_value_tracking_of_persisted_instance(self): + # Check initial internal states. + assert self.instance.get_changed_columns() == [] + assert self.instance._values['count'].previous_value == 0 + + # Change value and check internal states. + self.instance.count = 1 + assert self.instance.get_changed_columns() == ['count'] + assert self.instance._values['count'].previous_value == 0 + + # Internal states should be updated on save. + self.instance.save() + assert self.instance.get_changed_columns() == [] + assert self.instance._values['count'].previous_value == 1 + + # Change value twice. + self.instance.count = 2 + assert self.instance.get_changed_columns() == ['count'] + assert self.instance._values['count'].previous_value == 1 + self.instance.count = 3 + assert self.instance.get_changed_columns() == ['count'] + assert self.instance._values['count'].previous_value == 1 + + # Internal states updated on save. + self.instance.save() + assert self.instance.get_changed_columns() == [] + assert self.instance._values['count'].previous_value == 3 + + # Change value and reset it. + self.instance.count = 2 + assert self.instance.get_changed_columns() == ['count'] + assert self.instance._values['count'].previous_value == 3 + self.instance.count = 3 + assert self.instance.get_changed_columns() == [] + assert self.instance._values['count'].previous_value == 3 + + # Nothing to save: values in initial conditions. + self.instance.save() + assert self.instance.get_changed_columns() == [] + assert self.instance._values['count'].previous_value == 3 + + def test_previous_value_tracking_on_instanciation(self): + self.instance = TestMultiKeyModel( + partition=random.randint(0, 1000), + cluster=random.randint(0, 1000), + count=0, + text='happy') + + # Columns of instances not persisted yet should be marked as changed. + assert set(self.instance.get_changed_columns()) == set([ + 'partition', 'cluster', 'count', 'text']) + assert self.instance._values['partition'].previous_value is None + assert self.instance._values['cluster'].previous_value is None + assert self.instance._values['count'].previous_value is None + assert self.instance._values['text'].previous_value is None + + # Value changes doesn't affect internal states. + self.instance.count = 1 + assert 'count' in self.instance.get_changed_columns() + assert self.instance._values['count'].previous_value is None + self.instance.count = 2 + assert 'count' in self.instance.get_changed_columns() + assert self.instance._values['count'].previous_value is None + + # Value reset is properly tracked. + self.instance.count = None + assert 'count' not in self.instance.get_changed_columns() + assert self.instance._values['count'].previous_value == None + + self.instance.save() + assert self.instance.get_changed_columns() == [] + assert self.instance._values['count'].previous_value == None + assert self.instance.count is None + class TestCanUpdate(BaseCassEngTestCase): From 03d20d28cdb04f64035165a08bb93c203bc1a4ba Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 22 Jul 2015 11:58:49 -0500 Subject: [PATCH 0282/2431] don't build embedded extensions for gevent in tox config was failing on these for some reason also, not bothering with eventlet dep since it can't run without monkey patching --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index ab2b18a16f..0f97b2416f 100644 --- a/tox.ini +++ b/tox.ini @@ -13,10 +13,11 @@ deps = {[base]deps} blist {env:CYTHON_DEP:} py26: unittest2 - py{26,27}: twisted - py{26,27}: eventlet py{26,27}: gevent + twisted setenv = USE_CASS_EXTERNAL=1 + LIBEV_EMBED=0 + CARES_EMBED=0 changedir = {envtmpdir} commands = nosetests --verbosity=2 --no-path-adjustment {toxinidir}/tests/unit/ nosetests --verbosity=2 --no-path-adjustment {toxinidir}/tests/integration/cqlengine From 94b8f4d15dd6def3d600e1e6a329a4f81d3984c5 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 22 Jul 2015 14:48:33 -0500 Subject: [PATCH 0283/2431] doc: Add smallint, tinyint to getting started types table --- docs/getting_started.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index fa5cda32ce..9e4384ce2f 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -212,6 +212,8 @@ following way: | | ``int`` | | ``int`` | | | ``long`` | | ``bigint`` | | | | ``varint`` | + | | | ``smallint`` | + | | | ``tinyint`` | | | | ``counter`` | +--------------------+-------------------------+ | ``decimal.Decimal``| ``decimal`` | From 33e9a99627878e448b8ca40dce71c8a61cb071c8 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 17 Jul 2015 13:34:48 -0500 Subject: [PATCH 0284/2431] Remove unused CassandraType init, validate --- cassandra/cqltypes.py | 61 +------------- tests/unit/test_types.py | 150 +--------------------------------- tests/unit/test_util_types.py | 147 +++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 209 deletions(-) create mode 100644 tests/unit/test_util_types.py diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 77fc2b911b..2d858e52df 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -213,21 +213,9 @@ class _CassandraType(object): of EmptyValue) will be returned. """ - def __init__(self, val): - self.val = self.validate(val) - def __repr__(self): return '<%s( %r )>' % (self.cql_parameterized_type(), self.val) - @staticmethod - def validate(val): - """ - Called to transform an input value into one of a suitable type - for this class. As an example, the BooleanType class uses this - to convert an incoming value to True or False. - """ - return val - @classmethod def from_binary(cls, byts, protocol_version): """ @@ -358,10 +346,6 @@ class BytesType(_CassandraType): typename = 'blob' empty_binary_ok = True - @staticmethod - def validate(val): - return bytearray(val) - @staticmethod def serialize(val, protocol_version): return six.binary_type(val) @@ -370,10 +354,6 @@ def serialize(val, protocol_version): class DecimalType(_CassandraType): typename = 'decimal' - @staticmethod - def validate(val): - return Decimal(val) - @staticmethod def deserialize(byts, protocol_version): scale = int32_unpack(byts[:4]) @@ -412,10 +392,6 @@ def serialize(uuid, protocol_version): class BooleanType(_CassandraType): typename = 'boolean' - @staticmethod - def validate(val): - return bool(val) - @staticmethod def deserialize(byts, protocol_version): return bool(int8_unpack(byts)) @@ -556,14 +532,9 @@ class CounterColumnType(LongType): class DateType(_CassandraType): typename = 'timestamp' - @classmethod - def validate(cls, val): - if isinstance(val, six.string_types): - val = cls.interpret_datestring(val) - return val - @staticmethod def interpret_datestring(val): + # not used internally. deprecate? if val[-5] in ('+', '-'): offset = (int(val[-4:-2]) * 3600 + int(val[-2:]) * 60) * int(val[-5] + '1') val = val[:-5] @@ -579,9 +550,6 @@ def interpret_datestring(val): else: raise ValueError("can't interpret %r as a date" % (val,)) - def my_timestamp(self): - return self.val - @staticmethod def deserialize(byts, protocol_version): timestamp = int64_unpack(byts) / 1000.0 @@ -633,12 +601,6 @@ class SimpleDateType(_CassandraType): # range (2^31). EPOCH_OFFSET_DAYS = 2 ** 31 - @classmethod - def validate(cls, val): - if not isinstance(val, util.Date): - val = util.Date(val) - return val - @staticmethod def deserialize(byts, protocol_version): days = uint32_unpack(byts) - SimpleDateType.EPOCH_OFFSET_DAYS @@ -668,12 +630,6 @@ def serialize(byts, protocol_version): class TimeType(_CassandraType): typename = 'time' - @classmethod - def validate(cls, val): - if not isinstance(val, util.Time): - val = util.Time(val) - return val - @staticmethod def deserialize(byts, protocol_version): return util.Time(int64_unpack(byts)) @@ -709,11 +665,6 @@ class VarcharType(UTF8Type): class _ParameterizedType(_CassandraType): - def __init__(self, val): - if not self.subtypes: - raise ValueError("%s type with no parameters can't be instantiated" % (self.typename,)) - _CassandraType.__init__(self, val) - @classmethod def deserialize(cls, byts, protocol_version): if not cls.subtypes: @@ -730,11 +681,6 @@ def serialize(cls, val, protocol_version): class _SimpleParameterizedType(_ParameterizedType): - @classmethod - def validate(cls, val): - subtype, = cls.subtypes - return cls.adapter([subtype.validate(subval) for subval in val]) - @classmethod def deserialize_safe(cls, byts, protocol_version): subtype, = cls.subtypes @@ -787,11 +733,6 @@ class MapType(_ParameterizedType): typename = 'map' num_subtypes = 2 - @classmethod - def validate(cls, val): - key_type, value_type = cls.subtypes - return dict((key_type.validate(k), value_type.validate(v)) for (k, v) in six.iteritems(val)) - @classmethod def deserialize_safe(cls, byts, protocol_version): key_type, value_type = cls.subtypes diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 77667705d0..18d40956c7 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -105,11 +105,6 @@ def test_lookup_casstype(self): self.assertRaises(ValueError, lookup_casstype, 'AsciiType~') - # TODO: Do a few more tests - # "I would say some parameterized and nested types would be good to test, - # like "MapType(AsciiType, IntegerType)" and "ReversedType(AsciiType)" - self.assertEqual(str(lookup_casstype(BooleanType(True))), str(BooleanType(True))) - def test_casstype_parameterized(self): self.assertEqual(LongType.cass_parameterized_type_with(()), 'LongType') self.assertEqual(LongType.cass_parameterized_type_with((), full=True), 'org.apache.cassandra.db.marshal.LongType') @@ -125,134 +120,7 @@ def test_datetype_from_string(self): # Ensure all formats can be parsed, without exception for format in cassandra.cqltypes.cql_timestamp_formats: date_string = str(datetime.datetime.now().strftime(format)) - cassandra.cqltypes.DateType(date_string) - - def test_simpledate(self): - """ - Test cassandra.cqltypes.SimpleDateType() construction - """ - Date = cassandra.util.Date - # from datetime - expected_dt = datetime.datetime(1492, 10, 12, 1, 1) - expected_date = Date(expected_dt) - self.assertEqual(str(expected_date), '1492-10-12') - - # from string - sd = SimpleDateType('1492-10-12') - self.assertEqual(sd.val, expected_date) - sd = SimpleDateType('+1492-10-12') - self.assertEqual(sd.val, expected_date) - - # Date - sd = SimpleDateType(expected_date) - self.assertEqual(sd.val, expected_date) - - # date - sd = SimpleDateType(datetime.date(expected_dt.year, expected_dt.month, expected_dt.day)) - self.assertEqual(sd.val, expected_date) - - # days - sd = SimpleDateType(0) - self.assertEqual(sd.val, Date(datetime.date(1970, 1, 1))) - sd = SimpleDateType(-1) - self.assertEqual(sd.val, Date(datetime.date(1969, 12, 31))) - sd = SimpleDateType(1) - self.assertEqual(sd.val, Date(datetime.date(1970, 1, 2))) - # limits - min_builtin = Date(datetime.date(1, 1, 1)) - max_builtin = Date(datetime.date(9999, 12, 31)) - self.assertEqual(SimpleDateType(min_builtin.days_from_epoch).val, min_builtin) - self.assertEqual(SimpleDateType(max_builtin.days_from_epoch).val, max_builtin) - # just proving we can construct with on offset outside buildin range - self.assertEqual(SimpleDateType(min_builtin.days_from_epoch - 1).val.days_from_epoch, - min_builtin.days_from_epoch - 1) - self.assertEqual(SimpleDateType(max_builtin.days_from_epoch + 1).val.days_from_epoch, - max_builtin.days_from_epoch + 1) - - # no contruct - self.assertRaises(ValueError, SimpleDateType, '-1999-10-10') - self.assertRaises(TypeError, SimpleDateType, 1.234) - - # str - date_str = '2015-03-16' - self.assertEqual(str(Date(date_str)), date_str) - # out of range - self.assertEqual(str(Date(2932897)), '2932897') - self.assertEqual(repr(Date(1)), 'Date(1)') - - # eq other types - self.assertEqual(Date(1234), 1234) - self.assertEqual(Date(1), datetime.date(1970, 1, 2)) - self.assertFalse(Date(2932897) == datetime.date(9999, 12, 31)) # date can't represent year > 9999 - self.assertEqual(Date(2932897), 2932897) - - def test_time(self): - """ - Test cassandra.cqltypes.TimeType() construction - """ - Time = cassandra.util.Time - one_micro = 1000 - one_milli = 1000 * one_micro - one_second = 1000 * one_milli - one_minute = 60 * one_second - one_hour = 60 * one_minute - - # from strings - tt = TimeType('00:00:00.000000001') - self.assertEqual(tt.val, 1) - tt = TimeType('00:00:00.000001') - self.assertEqual(tt.val, one_micro) - tt = TimeType('00:00:00.001') - self.assertEqual(tt.val, one_milli) - tt = TimeType('00:00:01') - self.assertEqual(tt.val, one_second) - tt = TimeType('00:01:00') - self.assertEqual(tt.val, one_minute) - tt = TimeType('01:00:00') - self.assertEqual(tt.val, one_hour) - tt = TimeType('01:00:00.') - self.assertEqual(tt.val, one_hour) - - tt = TimeType('23:59:59.1') - tt = TimeType('23:59:59.12') - tt = TimeType('23:59:59.123') - tt = TimeType('23:59:59.1234') - tt = TimeType('23:59:59.12345') - - tt = TimeType('23:59:59.123456') - self.assertEqual(tt.val, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro) - - tt = TimeType('23:59:59.1234567') - self.assertEqual(tt.val, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro + 700) - - tt = TimeType('23:59:59.12345678') - self.assertEqual(tt.val, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro + 780) - - tt = TimeType('23:59:59.123456789') - self.assertEqual(tt.val, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro + 789) - - # from int - tt = TimeType(12345678) - self.assertEqual(tt.val, 12345678) - - # from time - expected_time = datetime.time(12, 1, 2, 3) - tt = TimeType(expected_time) - self.assertEqual(tt.val, expected_time) - - # util.Time self equality - self.assertEqual(Time(1234), Time(1234)) - - # str - time_str = '12:13:14.123456789' - self.assertEqual(str(Time(time_str)), time_str) - self.assertEqual(repr(Time(1)), 'Time(1)') - - # no construct - self.assertRaises(ValueError, TimeType, '1999-10-10 11:11:11.1234') - self.assertRaises(TypeError, TimeType, 1.234) - self.assertRaises(ValueError, TimeType, 123456789000000) - self.assertRaises(TypeError, TimeType, datetime.datetime(2004, 12, 23, 11, 11, 1)) + cassandra.cqltypes.DateType.interpret_datestring(date_string) def test_cql_typename(self): """ @@ -309,12 +177,6 @@ class BarType(FooType): def test_empty_value(self): self.assertEqual(str(EmptyValue()), 'EMPTY') - def test_cassandratype_base(self): - cassandra_type = _CassandraType('randomvaluetocheck') - self.assertEqual(cassandra_type.val, 'randomvaluetocheck') - self.assertEqual(cassandra_type.validate('randomvaluetocheck2'), 'randomvaluetocheck2') - self.assertEqual(cassandra_type.val, 'randomvaluetocheck') - def test_datetype(self): now_time_seconds = time.time() now_datetime = datetime.datetime.utcfromtimestamp(now_time_seconds) @@ -325,14 +187,6 @@ def test_datetype(self): # same results serialized self.assertEqual(DateType.serialize(now_datetime, 0), DateType.serialize(now_timestamp, 0)) - # from timestamp - date_type = DateType(now_timestamp) - self.assertEqual(date_type.my_timestamp(), now_timestamp) - - # from datetime object - date_type = DateType(now_datetime) - self.assertEqual(date_type.my_timestamp(), now_datetime) - # deserialize # epoc expected = 0 @@ -346,8 +200,6 @@ def test_datetype(self): expected = -770172256 self.assertEqual(DateType.deserialize(int64_pack(1000 * expected), 0), datetime.datetime(1945, 8, 5, 23, 15, 44)) - self.assertRaises(ValueError, date_type.interpret_datestring, 'fakestring') - # work around rounding difference among Python versions (PYTHON-230) expected = 1424817268.274 self.assertEqual(DateType.deserialize(int64_pack(int(1000 * expected)), 0), datetime.datetime(2015, 2, 24, 22, 34, 28, 274000)) diff --git a/tests/unit/test_util_types.py b/tests/unit/test_util_types.py new file mode 100644 index 0000000000..7cf6c1f4ba --- /dev/null +++ b/tests/unit/test_util_types.py @@ -0,0 +1,147 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + +import datetime + +from cassandra.util import Date, Time + + +class DateTests(unittest.TestCase): + + def test_from_datetime(self): + expected_date = datetime.date(1492, 10, 12) + d = Date(expected_date) + self.assertEqual(str(d), str(expected_date)) + + def test_from_string(self): + expected_date = datetime.date(1492, 10, 12) + d = Date(expected_date) + sd = Date('1492-10-12') + self.assertEqual(sd, d) + sd = Date('+1492-10-12') + self.assertEqual(sd, d) + + def test_from_date(self): + expected_date = datetime.date(1492, 10, 12) + d = Date(expected_date) + self.assertEqual(d.date(), expected_date) + + def test_from_days(self): + sd = Date(0) + self.assertEqual(sd, Date(datetime.date(1970, 1, 1))) + sd = Date(-1) + self.assertEqual(sd, Date(datetime.date(1969, 12, 31))) + sd = Date(1) + self.assertEqual(sd, Date(datetime.date(1970, 1, 2))) + + def test_limits(self): + min_builtin = Date(datetime.date(1, 1, 1)) + max_builtin = Date(datetime.date(9999, 12, 31)) + self.assertEqual(Date(min_builtin.days_from_epoch), min_builtin) + self.assertEqual(Date(max_builtin.days_from_epoch), max_builtin) + # just proving we can construct with on offset outside buildin range + self.assertEqual(Date(min_builtin.days_from_epoch - 1).days_from_epoch, + min_builtin.days_from_epoch - 1) + self.assertEqual(Date(max_builtin.days_from_epoch + 1).days_from_epoch, + max_builtin.days_from_epoch + 1) + + def test_invalid_init(self): + self.assertRaises(ValueError, Date, '-1999-10-10') + self.assertRaises(TypeError, Date, 1.234) + + def test_str(self): + date_str = '2015-03-16' + self.assertEqual(str(Date(date_str)), date_str) + + def test_out_of_range(self): + self.assertEqual(str(Date(2932897)), '2932897') + self.assertEqual(repr(Date(1)), 'Date(1)') + + def test_equals(self): + self.assertEqual(Date(1234), 1234) + self.assertEqual(Date(1), datetime.date(1970, 1, 2)) + self.assertFalse(Date(2932897) == datetime.date(9999, 12, 31)) # date can't represent year > 9999 + self.assertEqual(Date(2932897), 2932897) + + +class TimeTests(unittest.TestCase): + + def test_units_from_string(self): + one_micro = 1000 + one_milli = 1000 * one_micro + one_second = 1000 * one_milli + one_minute = 60 * one_second + one_hour = 60 * one_minute + + tt = Time('00:00:00.000000001') + self.assertEqual(tt.nanosecond_time, 1) + tt = Time('00:00:00.000001') + self.assertEqual(tt.nanosecond_time, one_micro) + tt = Time('00:00:00.001') + self.assertEqual(tt.nanosecond_time, one_milli) + tt = Time('00:00:01') + self.assertEqual(tt.nanosecond_time, one_second) + tt = Time('00:01:00') + self.assertEqual(tt.nanosecond_time, one_minute) + tt = Time('01:00:00') + self.assertEqual(tt.nanosecond_time, one_hour) + tt = Time('01:00:00.') + self.assertEqual(tt.nanosecond_time, one_hour) + + tt = Time('23:59:59.123456') + self.assertEqual(tt.nanosecond_time, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro) + + tt = Time('23:59:59.1234567') + self.assertEqual(tt.nanosecond_time, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro + 700) + + tt = Time('23:59:59.12345678') + self.assertEqual(tt.nanosecond_time, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro + 780) + + tt = Time('23:59:59.123456789') + self.assertEqual(tt.nanosecond_time, 23 * one_hour + 59 * one_minute + 59 * one_second + 123 * one_milli + 456 * one_micro + 789) + + def test_micro_precision(self): + Time('23:59:59.1') + Time('23:59:59.12') + Time('23:59:59.123') + Time('23:59:59.1234') + Time('23:59:59.12345') + + def test_from_int(self): + tt = Time(12345678) + self.assertEqual(tt.nanosecond_time, 12345678) + + def test_from_time(self): + expected_time = datetime.time(12, 1, 2, 3) + tt = Time(expected_time) + self.assertEqual(tt, expected_time) + + def test_equals(self): + # util.Time self equality + self.assertEqual(Time(1234), Time(1234)) + + def test_str_repr(self): + time_str = '12:13:14.123456789' + self.assertEqual(str(Time(time_str)), time_str) + self.assertEqual(repr(Time(1)), 'Time(1)') + + def test_invalid_init(self): + self.assertRaises(ValueError, Time, '1999-10-10 11:11:11.1234') + self.assertRaises(TypeError, Time, 1.234) + self.assertRaises(ValueError, Time, 123456789000000) + self.assertRaises(TypeError, Time, datetime.datetime(2004, 12, 23, 11, 11, 1)) From 6056b36bc8b92cc106576af44c25ebae0a524eec Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 17 Jul 2015 14:29:25 -0500 Subject: [PATCH 0285/2431] for RoundRobin return iterator instead of materialized list Most of time, only first element will be consumed. --- cassandra/policies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index 328b6b82fb..8132f8ab8e 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -173,7 +173,7 @@ def make_query_plan(self, working_keyspace=None, query=None): length = len(hosts) if length: pos %= length - return list(islice(cycle(hosts), pos, pos + length)) + return islice(cycle(hosts), pos, pos + length) else: return [] From c4abf2031325df100e202f9e71e7d03b09d2080a Mon Sep 17 00:00:00 2001 From: GregBestland Date: Wed, 22 Jul 2015 16:18:54 -0500 Subject: [PATCH 0286/2431] Changing tests to be cross python compatible --- .../standard/test_custom_protocol_handler.py | 4 +++- tests/unit/test_concurrent.py | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/integration/standard/test_custom_protocol_handler.py b/tests/integration/standard/test_custom_protocol_handler.py index 63d4b1991a..61a23831b7 100644 --- a/tests/integration/standard/test_custom_protocol_handler.py +++ b/tests/integration/standard/test_custom_protocol_handler.py @@ -22,6 +22,8 @@ from cassandra.cluster import Cluster from tests.integration import use_singledc, PROTOCOL_VERSION, execute_until_pass from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, get_sample +from six import binary_type + import uuid @@ -74,7 +76,7 @@ def test_custom_raw_uuid_row_results(self): result_set = session.execute("SELECT schema_version FROM system.local") result = result_set.pop() raw_value = result.pop() - self.assertEqual(type(raw_value), str) + self.assertTrue(isinstance(raw_value, binary_type)) self.assertEqual(len(raw_value), 16) # Ensure that we get normal uuid back when we re-connect diff --git a/tests/unit/test_concurrent.py b/tests/unit/test_concurrent.py index 5e23f584a0..0bdb1f9e4d 100644 --- a/tests/unit/test_concurrent.py +++ b/tests/unit/test_concurrent.py @@ -86,14 +86,14 @@ class TimedCallableInvoker(threading.Thread): def __init__(self, handler, slowdown=False): super(TimedCallableInvoker, self).__init__() self.slowdown = slowdown - self._stop = threading.Event() + self._stopper = threading.Event() self.handler = handler def stop(self): - self._stop.set() + self._stopper.set() def stopped(self): - return self._stop.isSet() + return self._stopper.isSet() def run(self): while(not self.stopped()): @@ -101,11 +101,11 @@ def run(self): pending_callback = self.handler.get_next_callback() priority_num = pending_callback[0] if (priority_num % 10) == 0 and self.slowdown: - self._stop.wait(.1) + self._stopper.wait(.1) callback_args = pending_callback[1] fn, args, kwargs, time_added = callback_args fn(time_added, *args, **kwargs) - self._stop.wait(.001) + self._stopper.wait(.001) return From 8afd853ae984bd5ef0fd2369dbd7d2a7c604e01f Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 15:43:57 +0100 Subject: [PATCH 0287/2431] Add typecodes to module with Cython-compatible .pxd file --- cassandra/protocol.py | 34 ++-------------------- cassandra/typecodes.pxd | 28 +++++++++++++++++++ cassandra/typecodes.py | 62 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 cassandra/typecodes.pxd create mode 100644 cassandra/typecodes.py diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 41439334d9..a6ce22ec08 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -22,6 +22,7 @@ from six.moves import range import io +from cassandra import typecodes from cassandra import (Unavailable, WriteTimeout, ReadTimeout, WriteFailure, ReadFailure, FunctionFailure, AlreadyExists, InvalidRequest, Unauthorized, @@ -35,7 +36,7 @@ DoubleType, FloatType, Int32Type, InetAddressType, IntegerType, ListType, LongType, MapType, SetType, TimeUUIDType, - UTF8Type, UUIDType, UserType, + UTF8Type, VarcharType, UUIDType, UserType, TupleType, lookup_casstype, SimpleDateType, TimeType, ByteType, ShortType) from cassandra.policies import WriteType @@ -531,35 +532,6 @@ def send_body(self, f, protocol_version): RESULT_KIND_PREPARED = 0x0004 RESULT_KIND_SCHEMA_CHANGE = 0x0005 -class CassandraTypeCodes(object): - CUSTOM_TYPE = 0x0000 - AsciiType = 0x0001 - LongType = 0x0002 - BytesType = 0x0003 - BooleanType = 0x0004 - CounterColumnType = 0x0005 - DecimalType = 0x0006 - DoubleType = 0x0007 - FloatType = 0x0008 - Int32Type = 0x0009 - UTF8Type = 0x000A - DateType = 0x000B - UUIDType = 0x000C - UTF8Type = 0x000D - IntegerType = 0x000E - TimeUUIDType = 0x000F - InetAddressType = 0x0010 - SimpleDateType = 0x0011 - TimeType = 0x0012 - ShortType = 0x0013 - ByteType = 0x0014 - ListType = 0x0020 - MapType = 0x0021 - SetType = 0x0022 - UserType = 0x0030 - TupleType = 0x0031 - - class ResultMessage(_MessageType): opcode = 0x08 name = 'RESULT' @@ -569,7 +541,7 @@ class ResultMessage(_MessageType): paging_state = None # Names match type name in module scope. Most are imported from cassandra.cqltypes (except CUSTOM_TYPE) - type_codes = _cqltypes_by_code = dict((v, globals()[k]) for k, v in CassandraTypeCodes.__dict__.items() if not k.startswith('_')) + type_codes = _cqltypes_by_code = dict((v, globals()[k]) for k, v in typecodes.__dict__.items() if not k.startswith('_')) _FLAGS_GLOBAL_TABLES_SPEC = 0x0001 _HAS_MORE_PAGES_FLAG = 0x0002 diff --git a/cassandra/typecodes.pxd b/cassandra/typecodes.pxd new file mode 100644 index 0000000000..b040528400 --- /dev/null +++ b/cassandra/typecodes.pxd @@ -0,0 +1,28 @@ +cdef enum: + CUSTOM_TYPE + AsciiType + LongType + BytesType + BooleanType + CounterColumnType + DecimalType + DoubleType + FloatType + Int32Type + UTF8Type + DateType + UUIDType + VarcharType + IntegerType + TimeUUIDType + InetAddressType + SimpleDateType + TimeType + ShortType + ByteType + ListType + MapType + SetType + UserType + TupleType + diff --git a/cassandra/typecodes.py b/cassandra/typecodes.py new file mode 100644 index 0000000000..651c58d765 --- /dev/null +++ b/cassandra/typecodes.py @@ -0,0 +1,62 @@ +""" +Module with constants for Cassandra type codes. + +These constants are useful for + + a) mapping messages to cqltypes (cassandra/cqltypes.py) + b) optimizezd dispatching for (de)serialization (cassandra/encoding.py) + +Type codes are repeated here from the Cassandra binary protocol specification: + + 0x0000 Custom: the value is a [string], see above. + 0x0001 Ascii + 0x0002 Bigint + 0x0003 Blob + 0x0004 Boolean + 0x0005 Counter + 0x0006 Decimal + 0x0007 Double + 0x0008 Float + 0x0009 Int + 0x000A Text + 0x000B Timestamp + 0x000C Uuid + 0x000D Varchar + 0x000E Varint + 0x000F Timeuuid + 0x0010 Inet + 0x0020 List: the value is an [option], representing the type + of the elements of the list. + 0x0021 Map: the value is two [option], representing the types of the + keys and values of the map + 0x0022 Set: the value is an [option], representing the type + of the elements of the set +""" + +CUSTOM_TYPE = 0x0000 +AsciiType = 0x0001 +LongType = 0x0002 +BytesType = 0x0003 +BooleanType = 0x0004 +CounterColumnType = 0x0005 +DecimalType = 0x0006 +DoubleType = 0x0007 +FloatType = 0x0008 +Int32Type = 0x0009 +UTF8Type = 0x000A +DateType = 0x000B +UUIDType = 0x000C +VarcharType = 0x000D +IntegerType = 0x000E +TimeUUIDType = 0x000F +InetAddressType = 0x0010 +SimpleDateType = 0x0011 +TimeType = 0x0012 +ShortType = 0x0013 +ByteType = 0x0014 +ListType = 0x0020 +MapType = 0x0021 +SetType = 0x0022 +UserType = 0x0030 +TupleType = 0x0031 + From f0b360a9c718b5d7c74604788a6092870242efcb Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 15:45:28 +0100 Subject: [PATCH 0288/2431] Cythonize marshalling code --- cassandra/marshal.pxd | 29 ++++++ cassandra/marshal.pyx | 201 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 cassandra/marshal.pxd create mode 100644 cassandra/marshal.pyx diff --git a/cassandra/marshal.pxd b/cassandra/marshal.pxd new file mode 100644 index 0000000000..ef7d9858f3 --- /dev/null +++ b/cassandra/marshal.pxd @@ -0,0 +1,29 @@ +from libc.stdint cimport (int8_t, int16_t, int32_t, int64_t, + uint8_t, uint16_t, uint32_t, uint64_t) + +cpdef bytes int64_pack(int64_t x) +cpdef bytes int32_pack(int32_t x) +cpdef bytes int16_pack(int16_t x) +cpdef bytes int8_pack(int8_t x) + +cpdef int64_t int64_unpack(const char *buf) +cpdef int32_t int32_unpack(const char *buf) +cpdef int16_t int16_unpack(const char *buf) +cpdef int8_t int8_unpack(const char *buf) + +cpdef bytes uint64_pack(uint64_t x) +cpdef bytes uint32_pack(uint32_t x) +cpdef bytes uint16_pack(uint16_t x) +cpdef bytes uint8_pack(uint8_t x) + +cpdef uint64_t uint64_unpack(const char *buf) +cpdef uint32_t uint32_unpack(const char *buf) +cpdef uint16_t uint16_unpack(const char *buf) +cpdef uint8_t uint8_unpack(const char *buf) + +cpdef bytes double_pack(double x) +cpdef bytes float_pack(float x) + +cpdef double double_unpack(const char *buf) +cpdef float float_unpack(const char *buf) + diff --git a/cassandra/marshal.pyx b/cassandra/marshal.pyx new file mode 100644 index 0000000000..4803686149 --- /dev/null +++ b/cassandra/marshal.pyx @@ -0,0 +1,201 @@ +# cython: profile=True +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import sys +import struct +import math + +from libc.stdint cimport (int8_t, int16_t, int32_t, int64_t, + uint8_t, uint16_t, uint32_t, uint64_t) + +assert sys.byteorder in ('little', 'big') +is_little_endian = sys.byteorder == 'little' + +# cdef extern from "marshal.h": +# cdef str c_string_to_python(char *p, Py_ssize_t len) + +def _make_packer(format_string): + packer = struct.Struct(format_string) + pack = packer.pack + unpack = lambda s: packer.unpack(s)[0] + return pack, unpack + + +cdef inline bytes pack(char *buf, Py_ssize_t size): + """ + Pack a buffer, given as a char *, into Python bytes in byte order. + """ + if is_little_endian: + swap_order(buf, size) + return buf[:size] + + +cdef inline swap_order(char *buf, Py_ssize_t size): + """ + Swap the byteorder of `buf` in-place (reverse all the bytes). + There are functions ntohl etc, but these may be POSIX-dependent. + """ + cdef Py_ssize_t start, end + cdef char c + for i in range(size/2): + end = size - i - 1 + c = buf[i] + buf[i] = buf[end] + buf[end] = c + +### Packing and unpacking of signed integers + +cpdef inline bytes int64_pack(int64_t x): + return pack( &x, 8) + +cpdef inline int64_t int64_unpack(const char *buf): + # The 'const' makes sure the buffer is not mutated in-place! + cdef int64_t x = ( buf)[0] + swap_order( &x, 8) + return x + +cpdef inline bytes int32_pack(int32_t x): + return pack( &x, 4) + +cpdef inline int32_t int32_unpack(const char *buf): + cdef int32_t x = ( buf)[0] + swap_order( &x, 4) + return x + +cpdef inline bytes int16_pack(int16_t x): + return pack( &x, 2) + +cpdef inline int16_t int16_unpack(const char *buf): + cdef int16_t x = ( buf)[0] + swap_order( &x, 2) + return x + +cpdef inline bytes int8_pack(int8_t x): + return ( &x)[:1] + +cpdef inline int8_t int8_unpack(const char *buf): + return ( buf)[0] + +cpdef inline bytes uint64_pack(uint64_t x): + return pack( &x, 8) + +cpdef inline uint64_t uint64_unpack(const char *buf): + cdef uint64_t x = ( buf)[0] + swap_order( &x, 8) + return x + +cpdef inline bytes uint32_pack(uint32_t x): + return pack( &x, 4) + +cpdef inline uint32_t uint32_unpack(const char *buf): + cdef uint32_t x = ( buf)[0] + swap_order( &x, 4) + return x + +cpdef inline bytes uint16_pack(uint16_t x): + return pack( &x, 2) + +cpdef inline uint16_t uint16_unpack(const char *buf): + cdef uint16_t x = ( buf)[0] + swap_order( &x, 2) + return x + +cpdef inline bytes uint8_pack(uint8_t x): + return pack( &x, 1) + +cpdef inline uint8_t uint8_unpack(const char *buf): + return ( buf)[0] + +cpdef inline bytes double_pack(double x): + return pack( &x, 8) + +cpdef inline double double_unpack(const char *buf): + cdef double x = ( buf)[0] + swap_order( &x, 8) + return x + +cpdef inline bytes float_pack(float x): + return pack( &x, 4) + +cpdef inline float float_unpack(const char *buf): + cdef float x = ( buf)[0] + swap_order( &x, 4) + return x + +# int64_pack, int64_unpack = _make_packer('>q') +# int32_pack, int32_unpack = _make_packer('>i') +# int16_pack, int16_unpack = _make_packer('>h') +# int8_pack, int8_unpack = _make_packer('>b') +# uint64_pack, uint64_unpack = _make_packer('>Q') +# uint32_pack, uint32_unpack = _make_packer('>I') +# uint16_pack, uint16_unpack = _make_packer('>H') +# uint8_pack, uint8_unpack = _make_packer('>B') +# float_pack, float_unpack = _make_packer('>f') +# double_pack, double_unpack = _make_packer('>d') + +# Special case for cassandra header +header_struct = struct.Struct('>BBbB') +header_pack = header_struct.pack +header_unpack = header_struct.unpack + +# in protocol version 3 and higher, the stream ID is two bytes +v3_header_struct = struct.Struct('>BBhB') +v3_header_pack = v3_header_struct.pack +v3_header_unpack = v3_header_struct.unpack + + +if six.PY3: + def varint_unpack(term): + val = int(''.join("%02x" % i for i in term), 16) + if (term[0] & 128) != 0: + # There is a bug in Cython (0.20 - 0.22), where if we do + # '1 << (len(term) * 8)' Cython generates '1' directly into the + # C code, causing integer overflows. Treat it as an object for now + val -= ( 1L) << (len(term) * 8) + return val +else: + def varint_unpack(term): # noqa + val = int(term.encode('hex'), 16) + if (ord(term[0]) & 128) != 0: + val = val - (1 << (len(term) * 8)) + return val + + +def bitlength(n): + # return int(math.log2(n)) + 1 + bitlen = 0 + while n > 0: + n >>= 1 + bitlen += 1 + return bitlen + + +def varint_pack(big): + pos = True + if big == 0: + return b'\x00' + if big < 0: + bytelength = bitlength(abs(big) - 1) // 8 + 1 + big = (1 << bytelength * 8) + big + pos = False + revbytes = bytearray() + while big > 0: + revbytes.append(big & 0xff) + big >>= 8 + if pos and revbytes[-1] & 0x80: + revbytes.append(0) + revbytes.reverse() + return six.binary_type(revbytes) From ad7e4e08481b8cd48c5724256c955604713101eb Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 15:47:04 +0100 Subject: [PATCH 0289/2431] Start on Cython version of ProtocolHandler --- cassandra/bytesio.pxd | 7 ++ cassandra/bytesio.pyx | 56 ++++++++++ cassandra/cython_protocol_handler.pyx | 154 ++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 cassandra/bytesio.pxd create mode 100644 cassandra/bytesio.pyx create mode 100644 cassandra/cython_protocol_handler.pyx diff --git a/cassandra/bytesio.pxd b/cassandra/bytesio.pxd new file mode 100644 index 0000000000..349fd600e6 --- /dev/null +++ b/cassandra/bytesio.pxd @@ -0,0 +1,7 @@ +cdef class BytesIOReader: + cdef bytes buf + cdef char *buf_ptr + cdef Py_ssize_t pos + cdef Py_ssize_t size + cdef char *read(self, Py_ssize_t n = ?) + diff --git a/cassandra/bytesio.pyx b/cassandra/bytesio.pyx new file mode 100644 index 0000000000..505fe391a6 --- /dev/null +++ b/cassandra/bytesio.pyx @@ -0,0 +1,56 @@ +# ython profile=True + +cdef class BytesIOReader: + """ + This class provides efficient support for reading bytes from a 'bytes' buffer, + by returning char * values directly without allocating intermediate objects. + """ + + def __init__(self, bytes buf): + self.buf = buf + self.size = len(buf) + self.buf_ptr = self.buf + + cdef char *read(self, Py_ssize_t n = -1): + """Read at most size bytes from the file + (less if the read hits EOF before obtaining size bytes). + + If the size argument is negative or omitted, read all data until EOF + is reached. The bytes are returned as a string object. An empty + string is returned when EOF is encountered immediately. + """ + cdef Py_ssize_t newpos = self.pos + n + cdef char *res + + if n < 0: + newpos = self.size + elif newpos > self.size: + self.pos = self.size + return b'' + else: + res = self.buf_ptr + self.pos + self.pos = newpos + return res + + +class PyBytesIOReader(BytesIOReader): + """ + Python-compatible BytesIOReader class + """ + + def read(self, n = -1): + """Read at most size bytes from the file + (less if the read hits EOF before obtaining size bytes). + + If the size argument is negative or omitted, read all data until EOF + is reached. The bytes are returned as a string object. An empty + string is returned when EOF is encountered immediately. + """ + if n is None or n < 0: + newpos = self.len + else: + newpos = min(self.pos+n, self.len) + r = self.buf[self.pos:newpos] + self.pos = newpos + return r + diff --git a/cassandra/cython_protocol_handler.pyx b/cassandra/cython_protocol_handler.pyx new file mode 100644 index 0000000000..85a5945a99 --- /dev/null +++ b/cassandra/cython_protocol_handler.pyx @@ -0,0 +1,154 @@ +# ython: profile=True + +from libc.stdint cimport int64_t, int32_t + +# from cassandra.marshal cimport (int8_pack, int8_unpack, int16_pack, int16_unpack, +# uint16_pack, uint16_unpack, uint32_pack, uint32_unpack, +# int32_pack, int32_unpack, int64_pack, int64_unpack, float_pack, float_unpack, double_pack, double_unpack) + +from cassandra.marshal import varint_pack, varint_unpack +from cassandra import util +from cassandra.cqltypes import EMPTY +from cassandra.protocol import ResultMessage, ProtocolHandler + +from cassandra.bytesio cimport BytesIOReader +from cassandra cimport typecodes + +import numpy as np + +include "marshal.pyx" + +class FastResultMessage(ResultMessage): + """ + Cython version of Result Message that has a faster implementation of + recv_results_row. + """ + # type_codes = ResultMessage.type_codes.copy() + code_to_type = dict((v, k) for k, v in ResultMessage.type_codes.items()) + + @classmethod + def recv_results_rows(cls, f, protocol_version, user_type_map): + paging_state, column_metadata = cls.recv_results_metadata(f, user_type_map) + + colnames = [c[2] for c in column_metadata] + coltypes = [c[3] for c in column_metadata] + colcodes = np.array( + [cls.code_to_type.get(coltype, -1) for coltype in coltypes], + dtype=np.dtype('i')) + parsed_rows = parse_rows(BytesIOReader(f.read()), colnames, + coltypes, colcodes, protocol_version) + return (paging_state, (colnames, parsed_rows)) + + +cdef parse_rows(BytesIOReader reader, list colnames, list coltypes, + int[::1] colcodes, protocol_version): + cdef Py_ssize_t i, rowcount + cdef char *raw_val + cdef int32_t raw_val_size + rowcount = read_int(reader) + # return RowIterator(reader, coltypes, colcodes, protocol_version, rowcount) + return [parse_row(reader, coltypes, colcodes, protocol_version) + for i in range(rowcount)] + + +cdef class RowIterator: + """ + Result iterator for a set of rows + + There seems to be an issue with generator expressions + memoryviews, so we + have a special iterator class instead. + """ + cdef list coltypes + cdef int[::1] colcodes + cdef Py_ssize_t rowcount, pos + cdef BytesIOReader reader + cdef object protocol_version + + def __init__(self, reader, coltypes, colcodes, protocol_version, rowcount): + self.reader = reader + self.coltypes = coltypes + self.colcodes = colcodes + self.protocol_version = protocol_version + self.rowcount = rowcount + self.pos = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.pos >= self.rowcount: + raise StopIteration + self.pos += 1 + return parse_row(self.reader, self.coltypes, self.colcodes, self.protocol_version) + + next = __next__ + + +cdef inline parse_row(BytesIOReader reader, list coltypes, int[::1] colcodes, + protocol_version): + cdef Py_ssize_t j + + row = [] + for j, ctype in enumerate(coltypes): + raw_val_size = read_int(reader) + if raw_val_size < 0: + val = None + else: + raw_val = reader.read(raw_val_size) + val = from_binary(ctype, colcodes[j], raw_val, + raw_val_size, protocol_version) + row.append(val) + + return row + + +class CythonProtocolHandler(ProtocolHandler): + """ + Use FastResultMessage to decode query result message messages. + """ + my_opcodes = ProtocolHandler.message_types_by_opcode.copy() + my_opcodes[FastResultMessage.opcode] = FastResultMessage + message_types_by_opcode = my_opcodes + + +cdef inline int32_t read_int(BytesIOReader reader): + return int32_unpack(reader.read(4)) + + +cdef inline from_binary(cqltype, int typecode, char *byts, int32_t size, protocol_version): + """ + Deserialize a bytestring into a value. See the deserialize() method + for more information. This method differs in that if None or the empty + string is passed in, None may be returned. + + This method provides a fast-path deserialization routine. + """ + if size == 0 and cqltype.empty_binary_ok: + return empty(cqltype) + return deserialize(cqltype, typecode, byts, size, protocol_version) + + +cdef empty(cqltype): + return EMPTY if cqltype.support_empty_values else None + + +def to_binary(cqltype, val, protocol_version): + """ + Serialize a value into a bytestring. See the serialize() method for + more information. This method differs in that if None is passed in, + the result is the empty string. + """ + return b'' if val is None else cqltype.serialize(val, protocol_version) + + +cdef deserialize(cqltype, int typecode, char *byts, int32_t size, protocol_version): + if typecode == typecodes.LongType: + return int64_unpack(byts) + else: + return deserialize_generic(cqltype, typecode, byts, size, protocol_version) + +cdef deserialize_generic(cqltype, int typecode, char *byts, int32_t size, + protocol_version): + print("deserialize", cqltype) + return cqltype.deserialize(byts[:size], protocol_version) + From 39af4e15698081348dc97acdcaa4ffcd284f6ae2 Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 16:03:23 +0100 Subject: [PATCH 0290/2431] Add Cython modules to setup.py --- cassandra/bytesio.pyx | 2 +- cassandra/cython_protocol_handler.pyx | 2 +- cassandra/marshal.pyx | 2 +- setup.py | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cassandra/bytesio.pyx b/cassandra/bytesio.pyx index 505fe391a6..82887f4383 100644 --- a/cassandra/bytesio.pyx +++ b/cassandra/bytesio.pyx @@ -1,4 +1,4 @@ -# ython profile=True +# -- cython profile=True cdef class BytesIOReader: """ diff --git a/cassandra/cython_protocol_handler.pyx b/cassandra/cython_protocol_handler.pyx index 85a5945a99..add1e9f5ad 100644 --- a/cassandra/cython_protocol_handler.pyx +++ b/cassandra/cython_protocol_handler.pyx @@ -1,4 +1,4 @@ -# ython: profile=True +# -- cython: profile=True from libc.stdint cimport int64_t, int32_t diff --git a/cassandra/marshal.pyx b/cassandra/marshal.pyx index 4803686149..0efbf705bd 100644 --- a/cassandra/marshal.pyx +++ b/cassandra/marshal.pyx @@ -1,4 +1,4 @@ -# cython: profile=True +# -- cython: profile=True # Copyright 2013-2015 DataStax, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/setup.py b/setup.py index 37899c2e01..7083d7aaac 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ DistutilsExecError) from distutils.cmd import Command - try: import subprocess has_subprocess = True @@ -262,11 +261,13 @@ def run_setup(extensions): if "--no-cython" not in sys.argv: try: from Cython.Build import cythonize - cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'marshal', 'metadata', 'pool', 'protocol', 'query', 'util'] + cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'metadata', 'pool', 'protocol', 'query', 'util'] compile_args = [] if is_windows else ['-Wno-unused-function'] extensions.extend(cythonize( [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], extra_compile_args=compile_args) for m in cython_candidates], exclude_failures=True)) + + extensions.extend(cythonize("cassandra/*.pyx")) except ImportError: sys.stderr.write("Cython is not installed. Not compiling core driver files as extensions (optional).") From 92457198cca1a3832455fbefcc81e3fed351b33d Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 16:18:19 +0100 Subject: [PATCH 0291/2431] Use return type void for swap_order --- cassandra/marshal.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/marshal.pyx b/cassandra/marshal.pyx index 0efbf705bd..8ffe3e46dd 100644 --- a/cassandra/marshal.pyx +++ b/cassandra/marshal.pyx @@ -43,7 +43,7 @@ cdef inline bytes pack(char *buf, Py_ssize_t size): return buf[:size] -cdef inline swap_order(char *buf, Py_ssize_t size): +cdef inline void swap_order(char *buf, Py_ssize_t size): """ Swap the byteorder of `buf` in-place (reverse all the bytes). There are functions ntohl etc, but these may be POSIX-dependent. From fe67aec185f63576e093d82a0491789a64301467 Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 16:27:17 +0100 Subject: [PATCH 0292/2431] Make sure swap_order uses no PyObjects --- cassandra/marshal.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cassandra/marshal.pyx b/cassandra/marshal.pyx index 8ffe3e46dd..529f45e75a 100644 --- a/cassandra/marshal.pyx +++ b/cassandra/marshal.pyx @@ -48,9 +48,9 @@ cdef inline void swap_order(char *buf, Py_ssize_t size): Swap the byteorder of `buf` in-place (reverse all the bytes). There are functions ntohl etc, but these may be POSIX-dependent. """ - cdef Py_ssize_t start, end + cdef Py_ssize_t start, end, i cdef char c - for i in range(size/2): + for i in range(size//2): end = size - i - 1 c = buf[i] buf[i] = buf[end] From 2b7997830a3073a2e73942d36f2e3b22f7443a6c Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 23 Jul 2015 16:31:44 +0100 Subject: [PATCH 0293/2431] Check endianness before byte-swapping --- cassandra/marshal.pyx | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cassandra/marshal.pyx b/cassandra/marshal.pyx index 529f45e75a..2ecb0fa590 100644 --- a/cassandra/marshal.pyx +++ b/cassandra/marshal.pyx @@ -22,7 +22,7 @@ from libc.stdint cimport (int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, uint32_t, uint64_t) assert sys.byteorder in ('little', 'big') -is_little_endian = sys.byteorder == 'little' +cdef bint is_little_endian = sys.byteorder == 'little' # cdef extern from "marshal.h": # cdef str c_string_to_python(char *p, Py_ssize_t len) @@ -38,23 +38,25 @@ cdef inline bytes pack(char *buf, Py_ssize_t size): """ Pack a buffer, given as a char *, into Python bytes in byte order. """ - if is_little_endian: - swap_order(buf, size) + swap_order(buf, size) return buf[:size] cdef inline void swap_order(char *buf, Py_ssize_t size): """ - Swap the byteorder of `buf` in-place (reverse all the bytes). + Swap the byteorder of `buf` in-place on little-endian platforms + (reverse all the bytes). There are functions ntohl etc, but these may be POSIX-dependent. """ cdef Py_ssize_t start, end, i cdef char c - for i in range(size//2): - end = size - i - 1 - c = buf[i] - buf[i] = buf[end] - buf[end] = c + + if is_little_endian: + for i in range(size//2): + end = size - i - 1 + c = buf[i] + buf[i] = buf[end] + buf[end] = c ### Packing and unpacking of signed integers From d3f7c17674f8263860c1b7acffb53629ac01c77e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 11:51:10 -0500 Subject: [PATCH 0294/2431] izip values in BoundStatement.bind small optimization --- cassandra/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/query.py b/cassandra/query.py index 21b668dfb3..0ba6c5947d 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -24,7 +24,7 @@ import struct import time import six -from six.moves import range +from six.moves import range, zip from cassandra import ConsistencyLevel, OperationTimedOut from cassandra.util import unix_time_from_uuid1 From c87159b6be399732e5614afcac4379cee8b8cf67 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 11:51:38 -0500 Subject: [PATCH 0295/2431] Optimize Connection._read_frame_header no seek, read in-place --- cassandra/connection.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 5db88985e3..59684e712b 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -39,7 +39,7 @@ from six.moves import range from cassandra import ConsistencyLevel, AuthenticationFailed, OperationTimedOut -from cassandra.marshal import int32_pack, uint8_unpack +from cassandra.marshal import int32_pack from cassandra.protocol import (ReadyMessage, AuthenticateMessage, OptionsMessage, StartupMessage, ErrorMessage, CredentialsMessage, QueryMessage, ResultMessage, ProtocolHandler, @@ -99,7 +99,16 @@ def decompress(byts): frame_header_v1_v2 = struct.Struct('>BbBi') frame_header_v3 = struct.Struct('>BhBi') -_Frame = namedtuple('Frame', ('version', 'flags', 'stream', 'opcode', 'body_offset', 'end_pos')) + +class _Frame(object): + def __init__(self, version, flags, stream, opcode, body_offset, end_pos): + self.version = version + self.flags = flags + self.stream = stream + self.opcode = opcode + self.body_offset = body_offset + self.end_pos = end_pos + NONBLOCKING = (errno.EAGAIN, errno.EWOULDBLOCK) @@ -442,24 +451,20 @@ def control_conn_disposed(self): @defunct_on_error def _read_frame_header(self): - buf = self._iobuf - pos = buf.tell() + buf = self._iobuf.getvalue() + pos = len(buf) if pos: - buf.seek(0) - version = uint8_unpack(buf.read(1)) & PROTOCOL_VERSION_MASK + version = ord(buf[0]) & PROTOCOL_VERSION_MASK if version > MAX_SUPPORTED_VERSION: raise ProtocolError("This version of the driver does not support protocol version %d" % version) frame_header = frame_header_v3 if version >= 3 else frame_header_v1_v2 # this frame header struct is everything after the version byte header_size = frame_header.size + 1 if pos >= header_size: - flags, stream, op, body_len = frame_header.unpack(buf.read(frame_header.size)) + flags, stream, op, body_len = frame_header.unpack_from(buf, 1) if body_len < 0: raise ProtocolError("Received negative body length: %r" % body_len) self._current_frame = _Frame(version, flags, stream, op, header_size, body_len + header_size) - - self._iobuf.seek(pos) - return pos def _reset_frame(self): From fad1535b5d37e4c5fadc0812c172fc81d749548d Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 20 Jul 2015 12:39:00 -0500 Subject: [PATCH 0296/2431] construct new IO buffer with remainder of previous --- cassandra/connection.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 59684e712b..8b52185d11 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -468,9 +468,7 @@ def _read_frame_header(self): return pos def _reset_frame(self): - leftover = self._iobuf.read() - self._iobuf = io.BytesIO() - self._iobuf.write(leftover) + self._iobuf = io.BytesIO(self._iobuf.read()) self._current_frame = None def process_io_buffer(self): From 5e69140c990f38371ae78da6147deb65e17e2bef Mon Sep 17 00:00:00 2001 From: GregBestland Date: Thu, 23 Jul 2015 19:18:02 -0500 Subject: [PATCH 0297/2431] Adding integration and unit tests for PYTHON-358 --- .../standard/test_control_connection.py | 74 +++++++++++++++++++ tests/unit/test_control_connection.py | 38 ++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tests/integration/standard/test_control_connection.py diff --git a/tests/integration/standard/test_control_connection.py b/tests/integration/standard/test_control_connection.py new file mode 100644 index 0000000000..af8ad6e0db --- /dev/null +++ b/tests/integration/standard/test_control_connection.py @@ -0,0 +1,74 @@ +# Copyright 2013-2015 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +# +# +# + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + + +from cassandra.cluster import Cluster +from cassandra.protocol import ConfigurationException +from tests.integration import use_singledc, PROTOCOL_VERSION +from tests.integration.datatype_utils import update_datatypes + + +def setup_module(): + use_singledc() + update_datatypes() + + +class ControlConnectionTests(unittest.TestCase): + def setUp(self): + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.session = self.cluster.connect() + + def tearDown(self): + try: + self.session.execute("DROP KEYSPACE keyspacetodrop ") + except (ConfigurationException): + # we already removed the keyspace. + pass + self.cluster.shutdown() + + def test_drop_keyspace(self): + """ + Test to validate that dropping a keyspace with user defined types doesn't kill the control connection. + + + Creates a keyspace, and populates with a user defined type. It then records the control_connection's id. It + will then drop the keyspace and get the id of the control_connection again. They should be the same. If they are + not dropping the keyspace likely caused the control connection to be rebuilt. + + @since 2.7.0 + @jira_ticket PYTHON-358 + @expected_result the control connection is not killed + + @test_category connection + """ + + self.session.execute(""" + CREATE KEYSPACE keyspacetodrop + WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1' } + """) + self.session.set_keyspace("keyspacetodrop") + self.session.execute("CREATE TYPE user (age int, name text)") + self.session.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen)") + cc_id_pre_drop = id(self.cluster.control_connection._connection) + self.session.execute("DROP KEYSPACE keyspacetodrop") + cc_id_post_drop = id(self.cluster.control_connection._connection) + self.assertEqual(cc_id_post_drop, cc_id_pre_drop) \ No newline at end of file diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 1429531c2d..f1fa9cb6c0 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -464,3 +464,41 @@ def test_refresh_disabled(self): cluster.scheduler.schedule_unique.assert_has_calls([call(ANY, cc_no_topo_refresh.refresh_node_list_and_token_map), call(0.0, cc_no_topo_refresh.refresh_schema, schema_event['keyspace'], schema_event['table'], None, None, None)]) + + +class EventTimingTest(unittest.TestCase): + """ + A simple test to validate that event scheduling happens in order + Added for PYTHON-358 + """ + def setUp(self): + self.cluster = MockCluster() + self.connection = MockConnection() + self.time = FakeTime() + + # Use 2 for the schema_event_refresh_window which is what we would normally default to. + self.control_connection = ControlConnection(self.cluster, 1, 2, 0) + self.control_connection._connection = self.connection + self.control_connection._time = self.time + + def test_event_delay_timing(self): + """ + Submits a wide array of events make sure that each is scheduled to occur in the order they were received + """ + prior_delay = 0 + for _ in range(100): + for change_type in ('CREATED', 'DROPPED', 'UPDATED'): + event = { + 'change_type': change_type, + 'keyspace': '1', + 'table': 'table1' + } + # This is to increment the fake time, we don't actually sleep here. + self.time.sleep(.001) + self.cluster.scheduler.reset_mock() + self.control_connection._handle_schema_change(event) + self.cluster.scheduler.mock_calls + # Grabs the delay parameter from the scheduler invocation + current_delay = self.cluster.scheduler.mock_calls[0][1][0] + self.assertLess(prior_delay, current_delay) + prior_delay = current_delay From 5a22bf816267b1e9713f75eabf0c666418de3be6 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 24 Jul 2015 11:46:57 -0500 Subject: [PATCH 0298/2431] connection._Frame equality function to fix unit test --- cassandra/connection.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 8b52185d11..e2fed44901 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -13,12 +13,14 @@ # limitations under the License. from __future__ import absolute_import # to enable import io from stdlib -from collections import defaultdict, deque, namedtuple +from collections import defaultdict, deque import errno from functools import wraps, partial from heapq import heappush, heappop import io import logging +import six +from six.moves import range import socket import struct import sys @@ -35,9 +37,6 @@ else: from six.moves.queue import Queue, Empty # noqa -import six -from six.moves import range - from cassandra import ConsistencyLevel, AuthenticationFailed, OperationTimedOut from cassandra.marshal import int32_pack from cassandra.protocol import (ReadyMessage, AuthenticateMessage, OptionsMessage, @@ -109,6 +108,16 @@ def __init__(self, version, flags, stream, opcode, body_offset, end_pos): self.body_offset = body_offset self.end_pos = end_pos + def __eq__(self, other): # facilitates testing + if isinstance(other, _Frame): + return (self.version == other.version and + self.flags == other.flags and + self.stream == other.stream and + self.opcode == other.opcode and + self.body_offset == other.body_offset and + self.end_pos == other.end_pos) + return NotImplemented + NONBLOCKING = (errno.EAGAIN, errno.EWOULDBLOCK) @@ -130,6 +139,7 @@ class ConnectionShutdown(ConnectionException): """ pass + class ProtocolVersionUnsupported(ConnectionException): """ Server rejected startup message due to unsupported protocol version @@ -139,6 +149,7 @@ def __init__(self, host, startup_version): (host, startup_version)) self.startup_version = startup_version + class ConnectionBusy(Exception): """ An attempt was made to send a message through a :class:`.Connection` that @@ -248,7 +259,7 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.connected_event = Event() @classmethod - def initialize_reactor(self): + def initialize_reactor(cls): """ Called once by Cluster.connect(). This should be used by implementations to set up any resources that will be shared across connections. @@ -256,7 +267,7 @@ def initialize_reactor(self): pass @classmethod - def handle_fork(self): + def handle_fork(cls): """ Called after a forking. This should cleanup any remaining reactor state from the parent process. From 4764b341d5a269e6183816e8f6823a437f38212f Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 24 Jul 2015 12:50:46 -0500 Subject: [PATCH 0299/2431] Fix Connection._read_fram_header update for Python3 --- cassandra/connection.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index e2fed44901..5112248b82 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -178,6 +178,11 @@ def wrapper(self, *args, **kwargs): DEFAULT_CQL_VERSION = '3.0.0' +if six.PY3: + def int_from_buf_item(i): + return i +else: + int_from_buf_item = ord class Connection(object): @@ -465,7 +470,7 @@ def _read_frame_header(self): buf = self._iobuf.getvalue() pos = len(buf) if pos: - version = ord(buf[0]) & PROTOCOL_VERSION_MASK + version = int_from_buf_item(buf[0]) & PROTOCOL_VERSION_MASK if version > MAX_SUPPORTED_VERSION: raise ProtocolError("This version of the driver does not support protocol version %d" % version) frame_header = frame_header_v3 if version >= 3 else frame_header_v1_v2 From 4a79c748319f6dc477f7bccf7cb4da42ce1f6d3e Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 24 Jul 2015 14:10:38 -0500 Subject: [PATCH 0300/2431] cython as optional extra for setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 37899c2e01..67594036e4 100644 --- a/setup.py +++ b/setup.py @@ -216,6 +216,7 @@ def run_setup(extensions): keywords='cassandra,cql,orm', include_package_data=True, install_requires=dependencies, + extras_require = {'cython': ['Cython']}, tests_require=['nose', 'mock<=1.0.1', 'PyYAML', 'pytz', 'sure'], classifiers=[ 'Development Status :: 5 - Production/Stable', From f2e9bb36b96947739c29f9392daa69292f563df7 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Fri, 24 Jul 2015 16:26:46 -0500 Subject: [PATCH 0301/2431] Upate performance notes doc: +cython, -numbers remove outdated, specific numbers Having numbers for contrived workloads sometimes confuses people or leads to impressions of false claims. Unified benchmarks at DataStax will replace these. --- docs/performance.rst | 299 ++++--------------------------------------- 1 file changed, 28 insertions(+), 271 deletions(-) diff --git a/docs/performance.rst b/docs/performance.rst index c7a6b3b290..cf11e73713 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -2,290 +2,47 @@ Performance Notes ================= The Python driver for Cassandra offers several methods for executing queries. You can synchronously block for queries to complete using -:meth:`.Session.execute()`, you can use a future-like interface through -:meth:`.Session.execute_async()`, or you can attach a callback to the future -with :meth:`.ResponseFuture.add_callback()`. Each of these methods has -different performance characteristics and behaves differently when -multiple threads are used. +:meth:`.Session.execute()`, you can obtain asynchronous request futures through +:meth:`.Session.execute_async()`, and you can attach a callback to the future +with :meth:`.ResponseFuture.add_callback()`. -Benchmark Notes ---------------- -All benchmarks were executed using the -`benchmark scripts `_ -in the driver repository. They were executed on a laptop with 16 GiB of RAM, an SSD, -and a 2 GHz, four core CPU with hyper-threading. The Cassandra cluster was a three -node `ccm `_ cluster running on the same laptop -with version 1.2.13 of Cassandra. I suggest testing these benchmarks against your -own cluster when tuning the driver for optimal throughput or latency. - -The 1.0.0 version of the driver was used with all default settings. For these -benchmarks, the driver was configured to use the ``libev`` reactor. You can also run -the benchmarks using the ``asyncore`` event loop (:class:`~.AsyncoreConnection`) -by using the ``--asyncore-only`` command line option. - -Each benchmark completes 100,000 small inserts. The replication factor for the -keyspace was three, so all nodes were replicas for the inserted rows. - -The benchmarks require the Python driver C extensions as well as a few additional -Python packages. Follow these steps to install the prerequisites: - -1. Install packages to support Python driver C extensions: - - * Debian/Ubuntu: ``sudo apt-get install gcc python-dev libev4 libev-dev`` - * RHEL/CentOS/Fedora: ``sudo yum install gcc python-dev libev4 libev-dev`` - -2. Install Python packages: ``pip install scales twisted blist`` -3. Re-install the Cassandra driver: ``pip install --upgrade cassandra-driver`` - -Synchronous Execution (`sync.py `_) -------------------------------------------------------------------------------------------------------------- -Although this is the simplest way to make queries, it has low throughput -in single threaded environments. This is basically what the benchmark -is doing: - -.. code-block:: python - - from cassandra.cluster import Cluster - - cluster = Cluster([127.0.0.1, 127.0.0.2, 127.0.0.3]) - session = cluster.connect() - - for i in range(100000): - session.execute("INSERT INTO mykeyspace.mytable (key, b, c) VALUES (a, 'b', 'c')") - -.. code-block:: bash - - ~/python-driver $ python benchmarks/sync.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 434.08/sec - - -This technique does scale reasonably well as we add more threads: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/sync.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=2 - Average throughput: 830.49/sec - ~/python-driver $ python benchmarks/sync.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=4 - Average throughput: 1078.27/sec - ~/python-driver $ python benchmarks/sync.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=8 - Average throughput: 1275.20/sec - ~/python-driver $ python benchmarks/sync.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=16 - Average throughput: 1345.56/sec - - -In my environment, throughput is maximized at about 20 threads. - - -Batched Futures (`future_batches.py `_) ---------------------------------------------------------------------------------------------------------------------------- -This is a simple way to work with futures for higher throughput. Essentially, -we start 120 queries asynchronously at the same time and then wait for them -all to complete. We then repeat this process until all 100,000 operations -have completed: - -.. code-block:: python - - futures = Queue.Queue(maxsize=121) - for i in range(100000): - if i % 120 == 0: - # clear the existing queue - while True: - try: - futures.get_nowait().result() - except Queue.Empty: - break - - future = session.execute_async(query) - futures.put_nowait(future) - -As expected, this improves throughput in a single-threaded environment: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/future_batches.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 3477.56/sec - -However, adding more threads may actually harm throughput: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/future_batches.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=2 - Average throughput: 2360.52/sec - ~/python-driver $ python benchmarks/future_batches.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=4 - Average throughput: 2293.21/sec - ~/python-driver $ python benchmarks/future_batches.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=8 - Average throughput: 2244.85/sec - - -Queued Futures (`future_full_pipeline.py `_) --------------------------------------------------------------------------------------------------------------------------------------- -This pattern is similar to batched futures. The main difference is that -every time we put a future on the queue, we pull the oldest future out -and wait for it to complete: - -.. code-block:: python - - futures = Queue.Queue(maxsize=121) - for i in range(100000): - if i >= 120: - old_future = futures.get_nowait() - old_future.result() - - future = session.execute_async(query) - futures.put_nowait(future) - -This gets slightly better throughput than the Batched Futures pattern: - -.. code-block:: bash +Examples of multiple request patterns can be found in the benchmark scripts included in the driver project. - ~/python-driver $ python benchmarks/future_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 3635.76/sec +The choice of execution pattern will depend on the application context. For applications dealing with multiple +requests in a given context, the recommended pattern is to use concurrent asynchronous +requests with callbacks. For many use cases, you don't need to implement this pattern yourself. +:meth:`cassandra.concurrent.execute_concurrent` and :meth:`cassandra.concurrent.execute_concurrent_with_args` +provide this pattern with a synchronous API and tunable concurrency. -But this has the same throughput issues when multiple threads are used: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/future_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=2 - Average throughput: 2213.62/sec - ~/python-driver $ python benchmarks/future_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=4 - Average throughput: 2707.62/sec - ~/python-driver $ python benchmarks/future_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=8 - Average throughput: 2462.42/sec - -Unthrottled Futures (`future_full_throttle.py `_) -------------------------------------------------------------------------------------------------------------------------------------------- -What happens if we don't throttle our async requests at all? - -.. code-block:: python - - futures = [] - for i in range(100000): - future = session.execute_async(query) - futures.append(future) - - for future in futures: - future.result() - -Throughput is about the same as the previous pattern, but a lot of memory will -be consumed by the list of Futures: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/future_full_throttle.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 3474.11/sec - ~/python-driver $ python benchmarks/future_full_throttle.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=2 - Average throughput: 2389.61/sec - ~/python-driver $ python benchmarks/future_full_throttle.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=4 - Average throughput: 2371.75/sec - ~/python-driver $ python benchmarks/future_full_throttle.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=8 - Average throughput: 2165.29/sec - -Callback Chaining (`callback_full_pipeline.py `_) ------------------------------------------------------------------------------------------------------------------------------------------------ -This pattern is very different from the previous patterns. Here we're taking -advantage of the :meth:`.ResponseFuture.add_callback()` function to start -another request as soon as one finishes. Furthermore, we're starting 120 -of these callback chains, so we've always got about 120 operations in -flight at any time: - -.. code-block:: python - - from itertools import count - from threading import Event - - sentinel = object() - num_queries = 100000 - num_started = count() - num_finished = count() - finished_event = Event() - - def insert_next(previous_result=sentinel): - if previous_result is not sentinel: - if isinstance(previous_result, BaseException): - log.error("Error on insert: %r", previous_result) - if num_finished.next() >= num_queries: - finished_event.set() - - if num_started.next() <= num_queries: - future = session.execute_async(query) - # NOTE: this callback also handles errors - future.add_callbacks(insert_next, insert_next) - - for i in range(min(120, num_queries)): - insert_next() - - finished_event.wait() - -This is a more complex pattern, but the throughput is excellent: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/callback_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 7647.30/sec - -Part of the reason why performance is so good is that everything is running on -single thread: the internal event loop thread that powers the driver. The -downside to this is that adding more threads doesn't improve anything: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/callback_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=2 - Average throughput: 7704.58/sec - - -What happens if we have more than 120 callback chains running? - -With 250 chains: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/callback_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 7794.22/sec - -Things look pretty good with 250 chains. If we try 500 chains, we start to max out -all of the connections in the connection pools. The problem is that the current -version of the driver isn't very good at throttling these callback chains, so -a lot of time gets spent waiting for new connections and performance drops -dramatically: - -.. code-block:: bash - - ~/python-driver $ python benchmarks/callback_full_pipeline.py -n 100000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --libev-only --threads=1 - Average throughput: 679.61/sec - -When :attr:`.Cluster.protocol_version` is set to 1 or 2, you should limit the -number of callback chains you run to roughly 100 per node in the cluster. -When :attr:`~.Cluster.protocol_version` is 3 or higher, you can safely experiment -with higher numbers of callback chains. - -For many use cases, you don't need to implement this pattern yourself. You can -simply use :meth:`cassandra.concurrent.execute_concurrent` and -:meth:`cassandra.concurrent.execute_concurrent_with_args`, which implement -this pattern for you with a synchronous API. +Due to the GIL and limited concurrency, the driver can become CPU-bound pretty quickly. The sections below +discuss further runtime and design considerations for mitigating this limitation. PyPy ---- -Almost all of these patterns become CPU-bound pretty quickly with CPython, the -normal implementation of python. `PyPy `_ is an alternative -implementation of Python (written in Python) which uses a JIT compiler to -reduce CPU consumption. This leads to a huge improvement in the driver -performance: +`PyPy `_ is an alternative Python runtime which uses a JIT compiler to +reduce CPU consumption. This leads to a huge improvement in the driver performance, +more than doubling throughput for many workloads. -.. code-block:: bash +Cython Extensions +----------------- +`Cython `_ is an optimizing compiler and language that can be used to compile the core files and +optional extensions for the driver. Cython is not a strict dependency, but the extensions will be built by default +if cython is present in the python path. To include Cython as a requirement, invoke with the extra name ``cython``: - ~/python-driver $ pypy benchmarks/callback_full_pipeline.py -n 500000 --hosts=127.0.0.1,127.0.0.2,127.0.0.3 --asyncore-only --threads=1 - Average throughput: 18782.00/sec +.. code-block:: bash -Eventually the driver may add C extensions to reduce CPU consumption, which -would probably narrow the gap between the performance of CPython and PyPy. + $ pip install cassandra-driver[cython] multiprocessing --------------- -All of the patterns here may be used over multiple processes using the +All of the patterns discussed above may be used over multiple processes using the `multiprocessing `_ -module. Multiple processes will scale significantly better than multiple -threads will, so if high throughput is your goal, consider this option. +module. Multiple processes will scale better than multiple threads, so if high throughput is your goal, +consider this option. -Just be sure to **never share any** :class:`~.Cluster`, :class:`~.Session`, +Be sure to **never share any** :class:`~.Cluster`, :class:`~.Session`, **or** :class:`~.ResponseFuture` **objects across multiple processes**. These objects should all be created after forking the process, not before. + +For further discussion and simple examples using the driver with ``multiprocessing``, +see `this blog post `_. From 6a00db265bd25fe4e87d8b2164164b836bd7b7e2 Mon Sep 17 00:00:00 2001 From: Ash Hoover Date: Fri, 24 Jul 2015 15:45:45 -0700 Subject: [PATCH 0302/2431] Fix a typo in cqlengine query docs --- docs/api/cassandra/cqlengine/query.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/cassandra/cqlengine/query.rst b/docs/api/cassandra/cqlengine/query.rst index 8486011a98..df823704f3 100644 --- a/docs/api/cassandra/cqlengine/query.rst +++ b/docs/api/cassandra/cqlengine/query.rst @@ -6,7 +6,7 @@ QuerySet -------- QuerySet objects are typically obtained by calling :meth:`~.cassandra.cqlengine.models.Model.objects` on a model class. -The mehtods here are used to filter, order, and constrain results. +The methods here are used to filter, order, and constrain results. .. autoclass:: ModelQuerySet From 8c79bc5d6030e0371d9ece42930d97fb84a64231 Mon Sep 17 00:00:00 2001 From: Kishan Karunaratne Date: Mon, 27 Jul 2015 16:35:48 -0700 Subject: [PATCH 0303/2431] added doxyfile for Doxygen --- doxyfile_python | 2337 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2337 insertions(+) create mode 100644 doxyfile_python diff --git a/doxyfile_python b/doxyfile_python new file mode 100644 index 0000000000..01cf58b6e0 --- /dev/null +++ b/doxyfile_python @@ -0,0 +1,2337 @@ +# Doxyfile 1.8.8 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Python Driver" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify an logo or icon that is included in +# the documentation. The maximum height of the logo should not exceed 55 pixels +# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo +# to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = /home/jenkins/workspace/python_docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = NO + +# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = NO + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a +# new page for each member. If set to NO, the documentation of a member will be +# part of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = +ALIASES += expected_errors="\par Expected Errors\n" +ALIASES += jira_ticket="\par JIRA Ticket\n" +ALIASES += expected_result="\par Expected Result\n" +ALIASES += test_assumptions="\par Test Assumptions\n" +ALIASES += note="\par Note\n" +ALIASES += test_category="\par Test Category\n" + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = YES + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by by putting a % sign in front of the word +# or globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO these classes will be included in the various overviews. This option has +# no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the +# todo list. This list is created by putting \todo commands in the +# documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the +# test list. This list is created by putting \test commands in the +# documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES the list +# will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO doxygen will only warn about wrong or incomplete parameter +# documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. +# Note: If this tag is empty the current directory is searched. + +INPUT = /home/jenkins/workspace/python_docs/tests/integration/ + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank the +# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii, +# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, +# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, +# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, +# *.qsf, *.as and *.js. + +FILE_PATTERNS = *.py + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = @Test + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. + +INPUT_FILTER = "python /usr/local/bin/doxypy.py" + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER ) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = YES + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES, then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefor more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra stylesheet files is of importance (e.g. the last +# stylesheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the stylesheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to NO can help when comparing the output of multiple runs. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler ( hhc.exe). If non-empty +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated ( +# YES) or that it should be included in the master .chm file ( NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated ( +# YES) or a normal table of contents ( NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using prerendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /