File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -64,21 +64,23 @@ def ingest_kafka(
6464 num_chunks = max (dataframe .shape [0 ] / max (chunk_size , 100 ), 1 )
6565 df_chunks = np .array_split (dataframe , num_chunks )
6666
67+ # Create queue through which encoding and ingestion will coordinate
68+ chunk_queue = Queue ()
69+
70+ # Start ingestion process to push feature rows to Kafka
71+ ingestion_process = Process (
72+ target = _kafka_feature_row_chunk_producer ,
73+ args = (
74+ chunk_queue ,
75+ num_chunks ,
76+ producer ,
77+ feature_set .get_kafka_source_topic (),
78+ progress_bar ,
79+ ),
80+ )
81+
6782 try :
68- # Create queue through which encoding and ingestion will coordinate
69- chunk_queue = Queue ()
70-
71- # Start ingestion process to push feature rows to Kafka
72- ingestion_process = Process (
73- target = _kafka_feature_row_chunk_producer ,
74- args = (
75- chunk_queue ,
76- num_chunks ,
77- producer ,
78- feature_set .get_kafka_source_topic (),
79- progress_bar ,
80- ),
81- )
83+ # Start ingestion process
8284 ingestion_process .start ()
8385
8486 # Create a pool of workers to convert df chunks into feature row chunks
@@ -99,7 +101,6 @@ def ingest_kafka(
99101 finally :
100102 producer .flush ()
101103 ingestion_process .join ()
102- ingestion_process .close ()
103104 rows_ingested = progress_bar .total
104105 progress_bar .close ()
105106 print (
You can’t perform that action at this time.
0 commit comments