From 22bcea576694b4fa47009b73d458760ce349f900 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 24 Feb 2020 16:43:47 +0800 Subject: [PATCH 001/378] kudu code scan --- .../java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java | 3 --- .../java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java | 3 --- .../com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java | 1 + .../java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java | 1 + .../com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java | 1 + 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 047665928..cf2c70f1f 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -227,9 +227,6 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { if (null != workerCount) { kuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 4a7457418..12689a07b 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -82,9 +82,6 @@ private void connKuDu() throws KuduException { if (null != workerCount) { asyncKuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - asyncKuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index c03c470a7..83864fa22 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -107,6 +107,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index c5cd8cdb2..91c94a2b2 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -190,6 +190,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info return KuduPredicate.newIsNotNullPredicate(column); case "IS NULL": return KuduPredicate.newIsNullPredicate(column); + default: } return null; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index ea61bd325..e38a85390 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -86,6 +86,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); From 86d05ee69c838e1d70e09719281006639c32e55e Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 15:01:30 +0800 Subject: [PATCH 002/378] mongo scan --- docs/mongoSide.md | 5 +- docs/mongoSink.md | 7 +-- .../flink/sql/side/mongo/MongoAllReqRow.java | 48 ++++---------- .../sql/side/mongo/MongoAsyncReqRow.java | 62 ++++++------------- .../flink/sql/side/mongo/utils/MongoUtil.java | 1 + .../sql/sink/mongo/MongoOutputFormat.java | 34 +--------- 6 files changed, 38 insertions(+), 119 deletions(-) diff --git a/docs/mongoSide.md b/docs/mongoSide.md index db557f441..73ce9644f 100644 --- a/docs/mongoSide.md +++ b/docs/mongoSide.md @@ -40,8 +40,6 @@ |----|---|---|----| | type |表明 输出表类型 mongo|是|| | address | 连接mongo数据库 jdbcUrl |是|| - | userName | mongo连接用户名|否|| - | password | mongo连接密码|否|| | tableName | mongo表名称|是|| | database | mongo表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| @@ -64,7 +62,8 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/docs/mongoSink.md b/docs/mongoSink.md index b0f916aab..b5bb6df0d 100644 --- a/docs/mongoSink.md +++ b/docs/mongoSink.md @@ -33,8 +33,6 @@ CREATE TABLE tableName( |----|----|----|----| |type |表明 输出表类型 mongo|是|| |address | 连接mongo数据库 jdbcUrl |是|| -|userName | mongo连接用户名|否|| -|password | mongo连接密码|否|| |tableName | mongo表名称|是|| |database | mongo表名称|是|| |parallelism | 并行度设置|否|1| @@ -46,9 +44,8 @@ CREATE TABLE MyResult( pv VARCHAR )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - userName ='dtstack', - password ='abc123', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='pv', parallelism ='1' diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index b416e4820..3229101c6 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -26,16 +26,13 @@ import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -49,7 +46,6 @@ import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -180,34 +176,13 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String userName, String password, String database, String tableName) { + private MongoCollection getConn(String address, String database, String tableName) { MongoCollection dbCollection; - try { - MongoCredential credential; - String[] servers = address.split(","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = server.split(":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - dbCollection = db.getCollection(tableName, Document.class); - return dbCollection; - } catch (Exception e) { - throw new RuntimeException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); + dbCollection = db.getCollection(tableName, Document.class); + return dbCollection; + } private void loadData(Map>> tmpCache) throws SQLException { @@ -217,8 +192,7 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), - tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -237,9 +211,9 @@ private void loadData(Map>> tmpCache) throws SQ //load data from table String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); - BasicDBObject basicDBObject = new BasicDBObject(); + BasicDBObject basicDbObject = new BasicDBObject(); for (String selectField : sideFieldNames) { - basicDBObject.append(selectField, 1); + basicDbObject.append(selectField, 1); } BasicDBObject filterObject = new BasicDBObject(); try { @@ -256,7 +230,7 @@ private void loadData(Map>> tmpCache) throws SQ } - FindIterable findIterable = dbCollection.find(filterObject).projection(basicDBObject).limit(FETCH_SIZE); + FindIterable findIterable = dbCollection.find(filterObject).projection(basicDbObject).limit(FETCH_SIZE); MongoCursor mongoCursor = findIterable.iterator(); while (mongoCursor.hasNext()) { Document doc = mongoCursor.next(); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 8ac7d83ee..5ef06c80e 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -29,11 +29,12 @@ import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; import com.mongodb.Block; +import com.mongodb.ConnectionString; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; -import com.mongodb.async.client.MongoClientSettings; +import com.mongodb.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; @@ -70,13 +71,11 @@ public class MongoAsyncReqRow extends AsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); - private final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = 20; - private transient MongoClient mongoClient; private MongoDatabase db; - private MongoSideTableInfo MongoSideTableInfo; + private MongoSideTableInfo mongoSideTableInfo; public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -85,48 +84,25 @@ public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = server.split(":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - ClusterSettings clusterSettings = ClusterSettings.builder().hosts(lists).build(); - ConnectionPoolSettings connectionPoolSettings = ConnectionPoolSettings.builder() - .maxSize(DEFAULT_MAX_DB_CONN_POOL_SIZE) + public void connMongoDb() throws Exception { + String address = mongoSideTableInfo.getAddress(); + ConnectionString connectionString = new ConnectionString(address); + + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connectionString) .build(); - if (!StringUtils.isEmpty(MongoSideTableInfo.getUserName()) || !StringUtils.isEmpty(MongoSideTableInfo.getPassword())) { - mongoCredential = MongoCredential.createCredential(MongoSideTableInfo.getUserName(), MongoSideTableInfo.getDatabase(), - MongoSideTableInfo.getPassword().toCharArray()); - MongoClientSettings settings = MongoClientSettings.builder().credential(mongoCredential) - .clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } else { - MongoClientSettings settings = MongoClientSettings.builder().clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } - db = mongoClient.getDatabase(MongoSideTableInfo.getDatabase()); + mongoClient = MongoClients.create(settings); + db = mongoClient.getDatabase(mongoSideTableInfo.getDatabase()); } @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - BasicDBObject basicDBObject = new BasicDBObject(); + BasicDBObject basicDbObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputCopy.row().getField(conValIndex); @@ -134,14 +110,14 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce dealMissKey(inputCopy, resultFuture); return; } - basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); + basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); } try { // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); if (null != filterCondition) { - basicDBObject.append(info.getFieldName(), filterCondition); + basicDbObject.append(info.getFieldName(), filterCondition); } return info; }).count(); @@ -149,7 +125,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDBObject.values()); + String key = buildCacheKey(basicDbObject.values()); if (openCache()) { CacheObj val = getFromCache(key); if (val != null) { @@ -171,7 +147,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } AtomicInteger atomicInteger = new AtomicInteger(0); - MongoCollection dbCollection = db.getCollection(MongoSideTableInfo.getTableName(), Document.class); + MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); Block printDocumentBlock = new Block() { @Override @@ -197,7 +173,7 @@ public void onResult(final Void result, final Throwable t) { } } }; - dbCollection.find(basicDBObject).forEach(printDocumentBlock, callbackWhenFinished); + dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); } @Override diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 6e0177759..4a9b77b9f 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -55,6 +55,7 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { return new BasicDBObject("$exists", true); case "IS NULL": return new BasicDBObject("$exists", false); + default: } return null; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 82bc9c15f..daaded15e 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -21,13 +21,10 @@ import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -37,9 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; /** * Reason: @@ -124,30 +118,8 @@ public void close() { } private void establishConnection() { - try { - MongoCredential credential; - String[] servers = address.split(","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = server.split(":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - } catch (Exception e) { - throw new IllegalArgumentException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); } private MongoOutputFormat() { From 40460cb57da1ac1bc548112f65fade396976a615 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 15:56:10 +0800 Subject: [PATCH 003/378] rdb scan --- .../side/clickhouse/ClickhouseAllReqRow.java | 7 +- .../clickhouse/ClickhouseAsyncReqRow.java | 2 +- .../sql/sink/clickhouse/ClickhouseSink.java | 9 +- .../flink/sql/side/db2/Db2AllReqRow.java | 7 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 9 +- .../sql/side/impala/ImpalaAllReqRow.java | 7 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 2 +- .../flink/sql/sink/impala/ImpalaSink.java | 12 +- .../flink/sql/side/mysql/MysqlAllReqRow.java | 8 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 2 +- .../flink/sql/sink/mysql/MysqlSink.java | 6 +- .../sql/side/oracle/OracleAllReqRow.java | 8 +- .../sql/side/oracle/OracleAsyncReqRow.java | 2 +- .../flink/sql/sink/oracle/OracleSink.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 30 +- .../sql/side/polardb/PolardbAllSideInfo.java | 22 ++ .../sql/side/polardb/PolardbAsyncReqRow.java | 24 +- .../side/polardb/PolardbAsyncSideInfo.java | 22 ++ .../side/polardb/table/PolardbSideParser.java | 23 ++ .../flink/sql/sink/polardb/PolardbSink.java | 30 +- .../sink/polardb/table/PolardbSinkParser.java | 22 ++ .../side/postgresql/PostgresqlAllReqRow.java | 8 +- .../postgresql/PostgresqlAsyncReqRow.java | 2 +- .../sql/sink/postgresql/PostgresqlSink.java | 6 +- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 228 ------------ .../sql/side/rdb/async/RdbAsyncReqRow.java | 24 +- .../provider/DTC3P0DataSourceProvider.java | 10 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 1 + .../{RdbSink.java => AbstractRdbSink.java} | 18 +- .../flink/sql/sink/rdb/JDBCOptions.java | 22 +- .../rdb/format/AbstractJDBCOutputFormat.java | 10 +- .../rdb/format/JDBCUpsertOutputFormat.java | 15 +- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 9 +- .../sql/sink/rdb/writer/UpsertWriter.java | 338 ------------------ .../side/sqlserver/SqlserverAllReqRow.java | 16 +- .../side/sqlserver/SqlserverAllSideInfo.java | 6 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 8 +- .../sqlserver/SqlserverAsyncSideInfo.java | 7 +- .../sqlserver/table/SqlserverSideParser.java | 6 +- .../sql/sink/sqlserver/SqlserverSink.java | 6 +- 41 files changed, 296 insertions(+), 706 deletions(-) delete mode 100644 rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/{RdbSink.java => AbstractRdbSink.java} (90%) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index 68c0c7984..a2bf8add9 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -21,20 +21,17 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; -public class ClickhouseAllReqRow extends RdbAllReqRow { +public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 305d65118..3733d7350 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -58,7 +58,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); } } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index bbb6a6fe4..5f7e41f7b 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,14 +22,11 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class ClickhouseSink extends RdbSink implements IStreamSinkGener { +public class ClickhouseSink extends AbstractRdbSink implements IStreamSinkGener { public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -37,7 +34,7 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index a77201b80..e64e55987 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -21,9 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.google.common.collect.Maps; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * Date: 2019/11/20 @@ -40,7 +37,7 @@ * @author xiuzhu */ -public class Db2AllReqRow extends RdbAllReqRow { +public class Db2AllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(Db2AllReqRow.class); diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 3d85c6fbd..59571aa56 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -74,7 +74,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); } } diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 9942d4438..da1434ace 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,13 +1,10 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class DbSink extends RdbSink { +public class DbSink extends AbstractRdbSink { public DbSink() { super(new DbDialect()); @@ -15,7 +12,7 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 9cd8c9194..4118421ee 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -22,10 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; -import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +34,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * side operator with cache for all(period reload) @@ -46,7 +43,7 @@ * @author xiuzhu */ -public class ImpalaAllReqRow extends RdbAllReqRow { +public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 63d53b80d..1b983ca2c 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -66,7 +66,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); } public JsonObject getClientConfig() { diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 41fa00926..91b6532bf 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.hadoop.conf.Configuration; @@ -36,7 +36,7 @@ * @author xiuzhu */ -public class ImpalaSink extends RdbSink implements IStreamSinkGener { +public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { private ImpalaTableInfo impalaTableInfo; @@ -47,7 +47,7 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(getImpalaJdbcUrl()) + .setDbUrl(getImpalaJdbcUrl()) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) @@ -70,8 +70,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbURL; - StringBuffer urlBuffer = new StringBuffer(dbURL); + String newUrl = dbUrl; + StringBuffer urlBuffer = new StringBuffer(dbUrl); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index b6b7f45cb..8cf6ab597 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author xuchao */ -public class MysqlAllReqRow extends RdbAllReqRow { +public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -54,13 +54,13 @@ public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 6120767ef..2d2f5a72a 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -76,7 +76,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 8c1146ede..0a1749f04 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; @@ -33,7 +33,7 @@ * @author xuchao */ -public class MysqlSink extends RdbSink implements IStreamSinkGener { +public class MysqlSink extends AbstractRdbSink implements IStreamSinkGener { public MysqlSink() { super(new MySQLDialect()); @@ -42,7 +42,7 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 18d9ba045..5477514a5 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -36,7 +36,7 @@ /** * side operator with cache for all(period reload) */ -public class OracleAllReqRow extends RdbAllReqRow { +public class OracleAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(OracleAllReqRow.class); @@ -47,11 +47,11 @@ public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index f973ffee2..f6827f41b 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -68,6 +68,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index ee0239a1f..2257d2815 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class OracleSink extends RdbSink implements IStreamSinkGener { +public class OracleSink extends AbstractRdbSink implements IStreamSinkGener { public OracleSink() { super(new OracleDialect()); @@ -38,7 +38,7 @@ public OracleSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index f9e0950dd..a5eec511b 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -1,9 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -15,7 +32,12 @@ import java.util.List; import java.util.Map; -public class PolardbAllReqRow extends RdbAllReqRow { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -28,13 +50,13 @@ public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index 2802bd402..fa2ad63e7 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; @@ -8,6 +25,11 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAllSideInfo extends RdbAllSideInfo { public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 3cbb0b368..0f8f82738 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; @@ -16,6 +33,11 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(PolardbAsyncReqRow.class); @@ -48,6 +70,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index 0b47cf3f7..f6afb8da8 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; @@ -8,6 +25,11 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 348b3aed9..1afb661f0 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -1,3 +1,21 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; @@ -5,6 +23,11 @@ import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index 44d558140..c69f31e8f 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,11 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.sink.polardb; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; - -public class PolardbSink extends RdbSink { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbSink extends AbstractRdbSink { public PolardbSink() { super(new PolardbDialect()); } @@ -13,7 +35,7 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index 68eead938..a57089941 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; @@ -5,6 +22,11 @@ import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 6d68cfdca..1c9aecfe1 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author tcm */ -public class PostgresqlAllReqRow extends RdbAllReqRow { +public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -54,13 +54,13 @@ public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 02b333819..a26c8b229 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -68,7 +68,7 @@ public void open(Configuration parameters) throws Exception { vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index cf45a6bd9..bb731152c 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * @author maqi */ -public class PostgresqlSink extends RdbSink implements IStreamSinkGener { +public class PostgresqlSink extends AbstractRdbSink implements IStreamSinkGener { public PostgresqlSink() { super(new PostgresqlDialect()); } @@ -36,7 +36,7 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java deleted file mode 100644 index 48d2407aa..000000000 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.rdb.all; - -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import org.apache.calcite.sql.JoinType; -import org.apache.commons.collections.CollectionUtils; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -/** - * side operator with cache for all(period reload) - * Date: 2018/11/26 - * Company: www.dtstack.com - * - * @author maqi - */ - -public abstract class RdbAllReqRow extends AllReqRow { - - private static final long serialVersionUID = 2098635140857937718L; - - private static final Logger LOG = LoggerFactory.getLogger(RdbAllReqRow.class); - - private static final int CONN_RETRY_NUM = 3; - - private AtomicReference>>> cacheRef = new AtomicReference<>(); - - public RdbAllReqRow(SideInfo sideInfo) { - super(sideInfo); - } - - @Override - public Row fillData(Row input, Object sideInput) { - Map cacheInfo = (Map) sideInput; - Row row = new Row(sideInfo.getOutFieldInfoList().size()); - for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { - Object obj = input.getField(entry.getValue()); - boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); - - //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. - if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); - } - - row.setField(entry.getKey(), obj); - } - - for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { - if (cacheInfo == null) { - row.setField(entry.getKey(), null); - } else { - row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); - } - } - - return row; - } - - @Override - protected void initCache() throws SQLException { - Map>> newCache = Maps.newConcurrentMap(); - cacheRef.set(newCache); - loadData(newCache); - } - - @Override - protected void reloadCache() { - //reload cacheRef and replace to old cacheRef - Map>> newCache = Maps.newConcurrentMap(); - try { - loadData(newCache); - } catch (SQLException e) { - LOG.error("", e); - } - - cacheRef.set(newCache); - LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); - } - - - @Override - public void flatMap(CRow value, Collector out) throws Exception { - List inputParams = Lists.newArrayList(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = value.row().getField(conValIndex); - if (equalObj == null) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } - return; - } - inputParams.add(equalObj); - } - - String key = buildKey(inputParams); - List> cacheList = cacheRef.get().get(key); - if (CollectionUtils.isEmpty(cacheList)) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value.row(), null); - out.collect(new CRow(row, value.change())); - } else { - return; - } - - return; - } - - for (Map one : cacheList) { - out.collect(new CRow(fillData(value.row(), one), value.change())); - } - } - - private String buildKey(List equalValList) { - StringBuilder sb = new StringBuilder(""); - for (Object equalVal : equalValList) { - sb.append(equalVal).append("_"); - } - - return sb.toString(); - } - - private String buildKey(Map val, List equalFieldList) { - StringBuilder sb = new StringBuilder(""); - for (String equalField : equalFieldList) { - sb.append(val.get(equalField)).append("_"); - } - - return sb.toString(); - } - - public abstract Connection getConn(String dbURL, String userName, String password); - - - private void loadData(Map>> tmpCache) throws SQLException { - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - Connection connection = null; - - try { - for (int i = 0; i < CONN_RETRY_NUM; i++) { - try { - connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); - break; - } catch (Exception e) { - if (i == CONN_RETRY_NUM - 1) { - throw new RuntimeException("", e); - } - try { - String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); - LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); - } catch (InterruptedException e1) { - LOG.error("", e1); - } - } - - } - - //load data from table - String sql = sideInfo.getSqlCondition(); - Statement statement = connection.createStatement(); - statement.setFetchSize(getFetchSize()); - ResultSet resultSet = statement.executeQuery(sql); - String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); - String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); - while (resultSet.next()) { - Map oneRow = Maps.newHashMap(); - for (String fieldName : sideFieldNames) { - Object object = resultSet.getObject(fieldName.trim()); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, fields[fieldIndex]); - oneRow.put(fieldName.trim(), object); - } - - String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); - List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); - list.add(oneRow); - } - } catch (Exception e) { - LOG.error("", e); - } finally { - if (connection != null) { - connection.close(); - } - } - } - - public int getFetchSize() { - return 1000; - } - -} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 7123e257b..bf7f8f774 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,16 +20,12 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import io.vertx.core.AsyncResult; -import io.vertx.core.Handler; -import io.vertx.core.Vertx; -import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.jdbc.JDBCClient; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; @@ -41,8 +37,6 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -73,7 +67,7 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSQLClient; + private transient SQLClient rdbSqlClient; public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); @@ -113,7 +107,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - rdbSQLClient.getConnection(conn -> { + rdbSqlClient.getConnection(conn -> { if (conn.failed()) { //Treatment failures resultFuture.completeExceptionally(conn.cause()); @@ -195,8 +189,8 @@ public Row fillData(Row input, Object line) { @Override public void close() throws Exception { super.close(); - if (rdbSQLClient != null) { - rdbSQLClient.close(); + if (rdbSqlClient != null) { + rdbSqlClient.close(); } } @@ -211,8 +205,8 @@ public String buildCacheKey(JsonArray jsonArray) { return sb.toString(); } - public void setRdbSQLClient(SQLClient rdbSQLClient) { - this.rdbSQLClient = rdbSQLClient; + public void setRdbSqlClient(SQLClient rdbSqlClient) { + this.rdbSqlClient = rdbSqlClient; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java index fcca4f0ef..542258257 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java @@ -25,13 +25,19 @@ import javax.sql.DataSource; import java.beans.PropertyVetoException; import java.sql.SQLException; - +/** + * Date: 2019/9/17 + * Company: www.dtstack.com + * @author maqi + */ public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { @Override public DataSource getDataSource(JsonObject config) throws SQLException { String url = config.getString("url"); - if (url == null) throw new NullPointerException("url cannot be null"); + if (url == null) { + throw new NullPointerException("url cannot be null"); + } String driverClass = config.getString("driver_class"); String user = config.getString("user"); String password = config.getString("password"); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 1205c5521..49d163d4d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,7 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + default: } return obj; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java similarity index 90% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index c65696903..b7990f9f4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,10 +33,6 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import java.io.Serializable; -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; -import java.sql.Types; import java.util.Arrays; import java.util.List; @@ -48,11 +43,8 @@ * * @author maqi */ -public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - - protected String driverName; - - protected String dbURL; +public abstract class AbstractRdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String dbUrl; protected String userName; @@ -88,18 +80,18 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab protected String updateMode; - public RdbSink(JDBCDialect jdbcDialect) { + public AbstractRdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbURL = rdbTableInfo.getUrl(); + this.dbUrl = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index e9a4f391f..f476bfadf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -29,7 +29,7 @@ public class JDBCOptions { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -37,9 +37,9 @@ public class JDBCOptions { private String schema; private JDBCDialect dialect; - private JDBCOptions(String dbURL, String tableName, String driverName, String username, + private JDBCOptions(String dbUrl, String tableName, String driverName, String username, String password, String schema, JDBCDialect dialect) { - this.dbURL = dbURL; + this.dbUrl = dbUrl; this.tableName = tableName; this.driverName = driverName; this.username = username; @@ -48,8 +48,8 @@ private JDBCOptions(String dbURL, String tableName, String driverName, String us this.dialect = dialect; } - public String getDbURL() { - return dbURL; + public String getDbUrl() { + return dbUrl; } public String getTableName() { @@ -84,7 +84,7 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JDBCOptions) { JDBCOptions options = (JDBCOptions) o; - return Objects.equals(dbURL, options.dbURL) && + return Objects.equals(dbUrl, options.dbUrl) && Objects.equals(tableName, options.tableName) && Objects.equals(driverName, options.driverName) && Objects.equals(username, options.username) && @@ -100,7 +100,7 @@ public boolean equals(Object o) { * Builder of {@link JDBCOptions}. */ public static class Builder { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -152,8 +152,8 @@ public Builder setSchema(String schema) { /** * required, JDBC DB url. */ - public Builder setDBUrl(String dbURL) { - this.dbURL = dbURL; + public Builder setDbUrl(String dbUrl) { + this.dbUrl = dbUrl; return this; } @@ -163,7 +163,7 @@ public Builder setDialect(JDBCDialect dialect) { } public JDBCOptions build() { - checkNotNull(dbURL, "No dbURL supplied."); + checkNotNull(dbUrl, "No dbURL supplied."); checkNotNull(tableName, "No tableName supplied."); if (this.driverName == null) { @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); + return new JDBCOptions(dbUrl, tableName, driverName, username, password, schema, dialect); } } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 73f17d8fd..3378ff3f2 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -47,16 +47,16 @@ public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat protected final String username; protected final String password; - private final String drivername; + private final String driverName; protected final String dbURL; protected transient Connection connection; - public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { + public AbstractJDBCOutputFormat(String username, String password, String driverName, String dbUrl) { this.username = username; this.password = password; - this.drivername = drivername; - this.dbURL = dbURL; + this.driverName = driverName; + this.dbURL = dbUrl; } @Override @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(drivername, getClass().getClassLoader()); + JDBCUtils.forName(driverName, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index ff7b24979..b13b39537 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -25,7 +25,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; +import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -44,6 +44,7 @@ /** * An upsert OutputFormat for JDBC. + * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -83,7 +84,7 @@ public JDBCUpsertOutputFormat( long flushIntervalMills, boolean allReplace, String updateMode) { - super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbUrl()); this.schema = options.getSchema(); this.tableName = options.getTableName(); this.dialect = options.getDialect(); @@ -111,12 +112,12 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); - LOG.info("execute insert sql： {}", insertSQL); - System.out.println("execute insert sql :" + insertSQL); - jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); + String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); + LOG.info("execute insert sql： {}", insertSql); + System.out.println("execute insert sql :" + insertSql); + jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); } else { - jdbcWriter = UpsertWriter.create( + jdbcWriter = AbstractUpsertWriter.create( dialect, schema, tableName, fieldNames, fieldTypes, keyFields, partitionFields, getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 59eee0858..2aa7b01f9 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -34,6 +34,7 @@ /** * Just append record to jdbc, can not receive retract/delete message. + * @author maqi */ public class AppendOnlyWriter implements JDBCWriter { @@ -41,7 +42,7 @@ public class AppendOnlyWriter implements JDBCWriter { private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); - private final String insertSQL; + private final String insertSql; private final int[] fieldTypes; private transient PreparedStatement statement; @@ -49,8 +50,8 @@ public class AppendOnlyWriter implements JDBCWriter { // only use metric private transient DtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { - this.insertSQL = insertSQL; + public AppendOnlyWriter(String insertSql, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { + this.insertSql = insertSql; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; } @@ -58,7 +59,7 @@ public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat m @Override public void open(Connection connection) throws SQLException { this.rows = new ArrayList(); - this.statement = connection.prepareStatement(insertSQL); + this.statement = connection.prepareStatement(insertSql); } /** diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java deleted file mode 100644 index c7074de59..000000000 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.rdb.writer; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; -import static org.apache.flink.util.Preconditions.checkNotNull; - -/** - * Upsert writer to deal with upsert, delete message.dd - */ -public abstract class UpsertWriter implements JDBCWriter { - - private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); - - - public static UpsertWriter create( - JDBCDialect dialect, - String schema, - String tableName, - String[] fieldNames, - int[] fieldTypes, - String[] keyFields, - String[] partitionFields, - boolean objectReuse, - boolean allReplace, - DtRichOutputFormat metricOutputFormat) { - - checkNotNull(keyFields); - - List nameList = Arrays.asList(fieldNames); - int[] pkFields = Arrays.stream(keyFields).mapToInt(nameList::indexOf).toArray(); - int[] pkTypes = fieldTypes == null ? null : - Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - - String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSQL); - System.out.println("deleteSQL is :" + deleteSQL); - - Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); - LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); - System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); - - return upsertSQL.map((Function) sql -> - new UpsertWriterUsingUpsertStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) - .orElseGet(() -> - new UpsertWriterUsingInsertUpdateStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, - dialect.getRowExistsStatement(tableName, keyFields), - dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), - dialect.getUpdateStatement(tableName, fieldNames, keyFields), - metricOutputFormat)); - } - - final int[] fieldTypes; - final int[] pkTypes; - private final int[] pkFields; - private final String deleteSQL; - private final boolean objectReuse; - - private transient Map> keyToRows; - private transient PreparedStatement deleteStatement; - // only use metric - private transient DtRichOutputFormat metricOutputFormat; - - private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { - this.fieldTypes = fieldTypes; - this.pkFields = pkFields; - this.pkTypes = pkTypes; - this.deleteSQL = deleteSQL; - this.objectReuse = objectReuse; - this.metricOutputFormat = metricOutputFormat; - } - - @Override - public void open(Connection connection) throws SQLException { - this.keyToRows = new HashMap<>(); - this.deleteStatement = connection.prepareStatement(deleteSQL); - } - - @Override - public void addRecord(Tuple2 record) throws SQLException { - // we don't need perform a deep copy, because jdbc field are immutable object. - Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; - // add records to buffer - keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); - } - - @Override - public void executeBatch(Connection connection) throws SQLException { - try { - if (keyToRows.size() > 0) { - for (Map.Entry> entry : keyToRows.entrySet()) { - Row pk = entry.getKey(); - Tuple2 tuple = entry.getValue(); - if (tuple.f0) { - processOneRowInBatch(pk, tuple.f1); - } else { - setRecordToStatement(deleteStatement, pkTypes, pk); - deleteStatement.addBatch(); - } - } - internalExecuteBatch(); - deleteStatement.executeBatch(); - connection.commit(); - keyToRows.clear(); - } - } catch (Exception e) { - // 清理批处理中的正确字段，防止重复写入 - connection.rollback(); - connection.commit(); - cleanBatchWhenError(); - executeUpdate(connection); - } - } - - @Override - public void executeUpdate(Connection connection) throws SQLException { - if (keyToRows.size() > 0) { - for (Map.Entry> entry : keyToRows.entrySet()) { - try { - Row pk = entry.getKey(); - Tuple2 tuple = entry.getValue(); - if (tuple.f0) { - processOneRowInBatch(pk, tuple.f1); - internalExecuteBatch(); - } else { - setRecordToStatement(deleteStatement, pkTypes, pk); - deleteStatement.executeUpdate(); - } - connection.commit(); - } catch (Exception e) { - System.out.println(e.getCause()); - // deal pg error: current transaction is aborted, commands ignored until end of transaction block - connection.rollback(); - connection.commit(); - if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ,this row is {}", entry.getValue()); - LOG.error("", e); - } - metricOutputFormat.outDirtyRecords.inc(); - } - } - keyToRows.clear(); - } - } - - abstract void processOneRowInBatch(Row pk, Row row) throws SQLException; - - abstract void internalExecuteBatch() throws SQLException; - - @Override - public void close() throws SQLException { - if (deleteStatement != null) { - deleteStatement.close(); - deleteStatement = null; - } - } - - private Row getPrimaryKey(Row row) { - Row pks = new Row(pkFields.length); - for (int i = 0; i < pkFields.length; i++) { - pks.setField(i, row.getField(pkFields[i])); - } - return pks; - } - - // ---------------------------------------------------------------------------------------- - - private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { - - private static final long serialVersionUID = 1L; - private final String upsertSQL; - - private transient PreparedStatement upsertStatement; - - private UpsertWriterUsingUpsertStatement( - int[] fieldTypes, - int[] pkFields, - int[] pkTypes, - boolean objectReuse, - String deleteSQL, - String upsertSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.upsertSQL = upsertSQL; - } - - @Override - public void open(Connection connection) throws SQLException { - super.open(connection); - upsertStatement = connection.prepareStatement(upsertSQL); - } - - @Override - void processOneRowInBatch(Row pk, Row row) throws SQLException { - setRecordToStatement(upsertStatement, fieldTypes, row); - upsertStatement.addBatch(); - } - - @Override - public void cleanBatchWhenError() throws SQLException { - upsertStatement.clearBatch(); - upsertStatement.clearParameters(); - } - - @Override - void internalExecuteBatch() throws SQLException { - upsertStatement.executeBatch(); - } - - @Override - public void close() throws SQLException { - super.close(); - if (upsertStatement != null) { - upsertStatement.close(); - upsertStatement = null; - } - } - } - - private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { - - private static final long serialVersionUID = 1L; - private final String existSQL; - private final String insertSQL; - private final String updateSQL; - - private transient PreparedStatement existStatement; - private transient PreparedStatement insertStatement; - private transient PreparedStatement updateStatement; - - private UpsertWriterUsingInsertUpdateStatement( - int[] fieldTypes, - int[] pkFields, - int[] pkTypes, - boolean objectReuse, - String deleteSQL, - String existSQL, - String insertSQL, - String updateSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.existSQL = existSQL; - this.insertSQL = insertSQL; - this.updateSQL = updateSQL; - } - - @Override - public void open(Connection connection) throws SQLException { - super.open(connection); - existStatement = connection.prepareStatement(existSQL); - insertStatement = connection.prepareStatement(insertSQL); - updateStatement = connection.prepareStatement(updateSQL); - } - - @Override - void processOneRowInBatch(Row pk, Row row) throws SQLException { - setRecordToStatement(existStatement, pkTypes, pk); - ResultSet resultSet = existStatement.executeQuery(); - boolean exist = resultSet.next(); - resultSet.close(); - if (exist) { - // do update - setRecordToStatement(updateStatement, fieldTypes, row); - updateStatement.addBatch(); - } else { - // do insert - setRecordToStatement(insertStatement, fieldTypes, row); - insertStatement.addBatch(); - } - } - - @Override - public void cleanBatchWhenError() throws SQLException { - updateStatement.clearBatch(); - insertStatement.clearBatch(); - } - - @Override - void internalExecuteBatch() throws SQLException { - updateStatement.executeBatch(); - insertStatement.executeBatch(); - } - - @Override - public void close() throws SQLException { - super.close(); - if (existStatement != null) { - existStatement.close(); - existStatement = null; - } - if (insertStatement != null) { - insertStatement.close(); - insertStatement = null; - } - if (updateStatement != null) { - updateStatement.close(); - updateStatement = null; - } - } - } -} diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index 961539fde..af1d94171 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -21,21 +21,26 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.sql.Connection; import java.sql.DriverManager; import java.util.List; import java.util.Map; + /** - * side operator with cache for all(period reload) + * side operator with cache for all(period reload) + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi */ -public class SqlserverAllReqRow extends RdbAllReqRow { +public class SqlserverAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAllReqRow.class); @@ -46,13 +51,12 @@ public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List addParams = Maps.newHashMap(); - //addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index 2b9913e7e..8f0fc63a4 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -24,7 +24,11 @@ import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAllSideInfo extends RdbAllSideInfo { public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index cb4240ca6..070bfe334 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -33,7 +33,11 @@ import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAsyncReqRow.class); @@ -67,6 +71,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index 347ca9bf4..d3c833c96 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -23,10 +23,15 @@ import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import java.util.List; +import java.util.List; +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index 3631dcd8a..be8ebb152 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -21,7 +21,11 @@ import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index dbbeb820e..3c331cdd0 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class SqlserverSink extends RdbSink implements IStreamSinkGener { +public class SqlserverSink extends AbstractRdbSink implements IStreamSinkGener { public SqlserverSink() { super(new SqlserverDialect()); @@ -38,7 +38,7 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) From 36549fb01ed85eefa816e2de7c9a1adf46245e2f Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 15:58:56 +0800 Subject: [PATCH 004/378] rename --- .../side/rdb/all/AbstractRdbAllReqRow.java | 228 ++++++++++++ .../sink/rdb/writer/AbstractUpsertWriter.java | 339 ++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java new file mode 100644 index 000000000..aeb88f7a3 --- /dev/null +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.rdb.all; + +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; +import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.CollectionUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +/** + * side operator with cache for all(period reload) + * Date: 2018/11/26 + * Company: www.dtstack.com + * + * @author maqi + */ + +public abstract class AbstractRdbAllReqRow extends AllReqRow { + + private static final long serialVersionUID = 2098635140857937718L; + + private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbAllReqRow.class); + + private static final int CONN_RETRY_NUM = 3; + + private AtomicReference>>> cacheRef = new AtomicReference<>(); + + public AbstractRdbAllReqRow(SideInfo sideInfo) { + super(sideInfo); + } + + @Override + public Row fillData(Row input, Object sideInput) { + Map cacheInfo = (Map) sideInput; + Row row = new Row(sideInfo.getOutFieldInfoList().size()); + for (Map.Entry entry : sideInfo.getInFieldIndex().entrySet()) { + Object obj = input.getField(entry.getValue()); + boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); + + //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. + if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { + obj = ((Timestamp) obj).getTime(); + } + + row.setField(entry.getKey(), obj); + } + + for (Map.Entry entry : sideInfo.getSideFieldNameIndex().entrySet()) { + if (cacheInfo == null) { + row.setField(entry.getKey(), null); + } else { + row.setField(entry.getKey(), cacheInfo.get(entry.getValue())); + } + } + + return row; + } + + @Override + protected void initCache() throws SQLException { + Map>> newCache = Maps.newConcurrentMap(); + cacheRef.set(newCache); + loadData(newCache); + } + + @Override + protected void reloadCache() { + //reload cacheRef and replace to old cacheRef + Map>> newCache = Maps.newConcurrentMap(); + try { + loadData(newCache); + } catch (SQLException e) { + LOG.error("", e); + } + + cacheRef.set(newCache); + LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); + } + + + @Override + public void flatMap(CRow value, Collector out) throws Exception { + List inputParams = Lists.newArrayList(); + for (Integer conValIndex : sideInfo.getEqualValIndex()) { + Object equalObj = value.row().getField(conValIndex); + if (equalObj == null) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); + } + return; + } + inputParams.add(equalObj); + } + + String key = buildKey(inputParams); + List> cacheList = cacheRef.get().get(key); + if (CollectionUtils.isEmpty(cacheList)) { + if (sideInfo.getJoinType() == JoinType.LEFT) { + Row row = fillData(value.row(), null); + out.collect(new CRow(row, value.change())); + } else { + return; + } + + return; + } + + for (Map one : cacheList) { + out.collect(new CRow(fillData(value.row(), one), value.change())); + } + } + + private String buildKey(List equalValList) { + StringBuilder sb = new StringBuilder(""); + for (Object equalVal : equalValList) { + sb.append(equalVal).append("_"); + } + + return sb.toString(); + } + + private String buildKey(Map val, List equalFieldList) { + StringBuilder sb = new StringBuilder(""); + for (String equalField : equalFieldList) { + sb.append(val.get(equalField)).append("_"); + } + + return sb.toString(); + } + + public abstract Connection getConn(String dbUrl, String userName, String password); + + + private void loadData(Map>> tmpCache) throws SQLException { + RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + Connection connection = null; + + try { + for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { + connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); + break; + } catch (Exception e) { + if (i == CONN_RETRY_NUM - 1) { + throw new RuntimeException("", e); + } + try { + String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); + } + } + + } + + //load data from table + String sql = sideInfo.getSqlCondition(); + Statement statement = connection.createStatement(); + statement.setFetchSize(getFetchSize()); + ResultSet resultSet = statement.executeQuery(sql); + String[] sideFieldNames = sideInfo.getSideSelectFields().split(","); + String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + while (resultSet.next()) { + Map oneRow = Maps.newHashMap(); + for (String fieldName : sideFieldNames) { + Object object = resultSet.getObject(fieldName.trim()); + int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); + object = SwitchUtil.getTarget(object, fields[fieldIndex]); + oneRow.put(fieldName.trim(), object); + } + + String cacheKey = buildKey(oneRow, sideInfo.getEqualFieldList()); + List> list = tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()); + list.add(oneRow); + } + } catch (Exception e) { + LOG.error("", e); + } finally { + if (connection != null) { + connection.close(); + } + } + } + + public int getFetchSize() { + return 1000; + } + +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java new file mode 100644 index 000000000..373a59f6a --- /dev/null +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.rdb.writer; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.types.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Upsert writer to deal with upsert, delete message.dd + * @author maqi + */ +public abstract class AbstractUpsertWriter implements JDBCWriter { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(AbstractUpsertWriter.class); + + + public static AbstractUpsertWriter create( + JDBCDialect dialect, + String schema, + String tableName, + String[] fieldNames, + int[] fieldTypes, + String[] keyFields, + String[] partitionFields, + boolean objectReuse, + boolean allReplace, + DtRichOutputFormat metricOutputFormat) { + + checkNotNull(keyFields); + + List nameList = Arrays.asList(fieldNames); + int[] pkFields = Arrays.stream(keyFields).mapToInt(nameList::indexOf).toArray(); + int[] pkTypes = fieldTypes == null ? null : + Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); + + String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSql); + System.out.println("deleteSQL is :" + deleteSql); + + Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); + System.out.println("execute UpsertStatement: " + upsertSql.orElse("use UsingInsertUpdateStatement")); + + return upsertSql.map((Function) sql -> + new UpsertWriterUsingUpsertStatement( + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, sql, metricOutputFormat)) + .orElseGet(() -> + new UpsertWriterUsingInsertUpdateStatement( + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, + dialect.getRowExistsStatement(tableName, keyFields), + dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), + dialect.getUpdateStatement(tableName, fieldNames, keyFields), + metricOutputFormat)); + } + + final int[] fieldTypes; + final int[] pkTypes; + private final int[] pkFields; + private final String deleteSql; + private final boolean objectReuse; + + private transient Map> keyToRows; + private transient PreparedStatement deleteStatement; + // only use metric + private transient DtRichOutputFormat metricOutputFormat; + + private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { + this.fieldTypes = fieldTypes; + this.pkFields = pkFields; + this.pkTypes = pkTypes; + this.deleteSql = deleteSql; + this.objectReuse = objectReuse; + this.metricOutputFormat = metricOutputFormat; + } + + @Override + public void open(Connection connection) throws SQLException { + this.keyToRows = new HashMap<>(); + this.deleteStatement = connection.prepareStatement(deleteSql); + } + + @Override + public void addRecord(Tuple2 record) throws SQLException { + // we don't need perform a deep copy, because jdbc field are immutable object. + Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; + // add records to buffer + keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + } + + @Override + public void executeBatch(Connection connection) throws SQLException { + try { + if (keyToRows.size() > 0) { + for (Map.Entry> entry : keyToRows.entrySet()) { + Row pk = entry.getKey(); + Tuple2 tuple = entry.getValue(); + if (tuple.f0) { + processOneRowInBatch(pk, tuple.f1); + } else { + setRecordToStatement(deleteStatement, pkTypes, pk); + deleteStatement.addBatch(); + } + } + internalExecuteBatch(); + deleteStatement.executeBatch(); + connection.commit(); + keyToRows.clear(); + } + } catch (Exception e) { + // 清理批处理中的正确字段，防止重复写入 + connection.rollback(); + connection.commit(); + cleanBatchWhenError(); + executeUpdate(connection); + } + } + + @Override + public void executeUpdate(Connection connection) throws SQLException { + if (keyToRows.size() > 0) { + for (Map.Entry> entry : keyToRows.entrySet()) { + try { + Row pk = entry.getKey(); + Tuple2 tuple = entry.getValue(); + if (tuple.f0) { + processOneRowInBatch(pk, tuple.f1); + internalExecuteBatch(); + } else { + setRecordToStatement(deleteStatement, pkTypes, pk); + deleteStatement.executeUpdate(); + } + connection.commit(); + } catch (Exception e) { + System.out.println(e.getCause()); + // deal pg error: current transaction is aborted, commands ignored until end of transaction block + connection.rollback(); + connection.commit(); + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ,this row is {}", entry.getValue()); + LOG.error("", e); + } + metricOutputFormat.outDirtyRecords.inc(); + } + } + keyToRows.clear(); + } + } + + abstract void processOneRowInBatch(Row pk, Row row) throws SQLException; + + abstract void internalExecuteBatch() throws SQLException; + + @Override + public void close() throws SQLException { + if (deleteStatement != null) { + deleteStatement.close(); + deleteStatement = null; + } + } + + private Row getPrimaryKey(Row row) { + Row pks = new Row(pkFields.length); + for (int i = 0; i < pkFields.length; i++) { + pks.setField(i, row.getField(pkFields[i])); + } + return pks; + } + + // ---------------------------------------------------------------------------------------- + + private static final class UpsertWriterUsingUpsertStatement extends AbstractUpsertWriter { + + private static final long serialVersionUID = 1L; + private final String upsertSql; + + private transient PreparedStatement upsertStatement; + + private UpsertWriterUsingUpsertStatement( + int[] fieldTypes, + int[] pkFields, + int[] pkTypes, + boolean objectReuse, + String deleteSql, + String upsertSql, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.upsertSql = upsertSql; + } + + @Override + public void open(Connection connection) throws SQLException { + super.open(connection); + upsertStatement = connection.prepareStatement(upsertSql); + } + + @Override + void processOneRowInBatch(Row pk, Row row) throws SQLException { + setRecordToStatement(upsertStatement, fieldTypes, row); + upsertStatement.addBatch(); + } + + @Override + public void cleanBatchWhenError() throws SQLException { + upsertStatement.clearBatch(); + upsertStatement.clearParameters(); + } + + @Override + void internalExecuteBatch() throws SQLException { + upsertStatement.executeBatch(); + } + + @Override + public void close() throws SQLException { + super.close(); + if (upsertStatement != null) { + upsertStatement.close(); + upsertStatement = null; + } + } + } + + private static final class UpsertWriterUsingInsertUpdateStatement extends AbstractUpsertWriter { + + private static final long serialVersionUID = 1L; + private final String existSql; + private final String insertSql; + private final String updateSql; + + private transient PreparedStatement existStatement; + private transient PreparedStatement insertStatement; + private transient PreparedStatement updateStatement; + + private UpsertWriterUsingInsertUpdateStatement( + int[] fieldTypes, + int[] pkFields, + int[] pkTypes, + boolean objectReuse, + String deleteSql, + String existSql, + String insertSql, + String updateSql, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.existSql = existSql; + this.insertSql = insertSql; + this.updateSql = updateSql; + } + + @Override + public void open(Connection connection) throws SQLException { + super.open(connection); + existStatement = connection.prepareStatement(existSql); + insertStatement = connection.prepareStatement(insertSql); + updateStatement = connection.prepareStatement(updateSql); + } + + @Override + void processOneRowInBatch(Row pk, Row row) throws SQLException { + setRecordToStatement(existStatement, pkTypes, pk); + ResultSet resultSet = existStatement.executeQuery(); + boolean exist = resultSet.next(); + resultSet.close(); + if (exist) { + // do update + setRecordToStatement(updateStatement, fieldTypes, row); + updateStatement.addBatch(); + } else { + // do insert + setRecordToStatement(insertStatement, fieldTypes, row); + insertStatement.addBatch(); + } + } + + @Override + public void cleanBatchWhenError() throws SQLException { + updateStatement.clearBatch(); + insertStatement.clearBatch(); + } + + @Override + void internalExecuteBatch() throws SQLException { + updateStatement.executeBatch(); + insertStatement.executeBatch(); + } + + @Override + public void close() throws SQLException { + super.close(); + if (existStatement != null) { + existStatement.close(); + existStatement = null; + } + if (insertStatement != null) { + insertStatement.close(); + insertStatement = null; + } + if (updateStatement != null) { + updateStatement.close(); + updateStatement = null; + } + } + } +} From aac3a273f52d39318e943c2a4616d4e5a97394e8 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 16:13:25 +0800 Subject: [PATCH 005/378] redis can --- .../flink/sql/side/redis/RedisAllReqRow.java | 7 ++-- .../sql/side/redis/RedisAllSideInfo.java | 4 ++- .../sql/side/redis/RedisAsyncReqRow.java | 5 ++- .../sql/side/redis/RedisAsyncSideInfo.java | 4 ++- .../sql/side/redis/table/RedisSideParser.java | 3 ++ .../side/redis/table/RedisSideTableInfo.java | 4 ++- .../sql/sink/redis/RedisOutputFormat.java | 35 +++++++++++++------ .../flink/sql/sink/redis/RedisSink.java | 4 ++- .../sql/sink/redis/table/RedisSinkParser.java | 5 +-- .../sql/sink/redis/table/RedisTableInfo.java | 4 ++- .../CustomerSocketTextStreamFunction.java | 2 +- .../table/ServersocketSourceTableInfo.java | 2 +- 12 files changed, 56 insertions(+), 23 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 59997f5cb..a215c9d75 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -39,7 +39,9 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; - +/** + * @author yanxi + */ public class RedisAllReqRow extends AllReqRow{ private static final long serialVersionUID = 7578879189085344807L; @@ -259,7 +261,8 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout,1, poolConfig); + jedis = new JedisCluster(addresses, timeout, timeout, 1, poolConfig); + default: } return jedis; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index 81e7b1f4f..fc24bc4ef 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -28,7 +28,9 @@ import com.google.common.collect.Lists; import java.util.List; - +/** + * @author yanxi + */ public class RedisAllSideInfo extends SideInfo { private static final long serialVersionUID = 1998703966487857613L; diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d20875e01..ee951a928 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -43,7 +43,9 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; - +/** + * @author yanxi + */ public class RedisAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -108,6 +110,7 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ clusterClient = RedisClusterClient.create(clusterUri.toString()); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); + default: } } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index 9bdefe8c5..2736fc8a7 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -28,7 +28,9 @@ import com.google.common.collect.Lists; import java.util.List; - +/** + * @author yanxi + */ public class RedisAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -4851348392924455039L; diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index f8a83b868..2d0fb77d4 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -24,6 +24,9 @@ import java.util.Map; +/** + * @author yanxi + */ public class RedisSideParser extends AbsSideTableParser { @Override diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 88cfcb8a2..ae6cdd017 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -20,7 +20,9 @@ import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; - +/** + * @author yanxi + */ public class RedisSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 62d14cabd..bbf49dff7 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -26,11 +26,23 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; + import java.io.Closeable; import java.io.IOException; -import java.util.*; - +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * @author yanxi + */ public class RedisOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); @@ -129,7 +141,8 @@ private void establishConnection() { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout,10, password, poolConfig); + jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + default: } } @@ -145,10 +158,10 @@ public void writeRecord(Tuple2 record) throws IOException { return; } - HashMap map = new HashMap<>(); - for (String primaryKey : primaryKeys){ - for (int i=0; i map = new HashMap<>(8); + for (String primaryKey : primaryKeys) { + for (int i = 0; i < fieldNames.length; i++) { + if (fieldNames[i].equals(primaryKey)) { map.put(primaryKey, i); } } @@ -156,10 +169,10 @@ public void writeRecord(Tuple2 record) throws IOException { List kvList = new LinkedList<>(); for (String primaryKey : primaryKeys){ - StringBuilder primaryKV = new StringBuilder(); + StringBuilder primaryKv = new StringBuilder(); int index = map.get(primaryKey).intValue(); - primaryKV.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKV.toString()); + primaryKv.append(primaryKey).append(":").append(row.getField(index)); + kvList.add(primaryKv.toString()); } String perKey = String.join(":", kvList); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index d2e28c01f..007c7fd3b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -33,7 +33,9 @@ import org.apache.flink.types.Row; import java.util.List; - +/** + * @author yanxi + */ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 49861bb9a..201abab2c 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -24,11 +24,12 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; - +/** + * @author yanxi + */ public class RedisSinkParser extends AbsTableParser { @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 60d6dd12f..5c9940252 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -20,7 +20,9 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; - +/** + * @author yanxi + */ public class RedisTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index e46e0a76f..2f508b3e5 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -94,7 +94,7 @@ public void run(SourceContext ctx) throws Exception { while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { String record = buffer.substring(0, delimPos); // truncate trailing carriage return - if (delimiter.equals("\n") && record.endsWith("\r")) { + if ("\n".equals(delimiter) && "\r".endsWith(record)) { record = record.substring(0, record.length() - 1); } try { diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 796728eb3..02f6e9f30 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -28,7 +28,7 @@ * @author maqi */ public class ServersocketSourceTableInfo extends SourceTableInfo { - //version + private static final String CURR_TYPE = "serversocket"; public static final String HOSTNAME_KEY = "host"; From 42db63ec3d88d2234b2415846619b40e703dd07f Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 25 Feb 2020 16:20:24 +0800 Subject: [PATCH 006/378] merge 1.8_dev --- core/src/main/java/com/dtstack/flink/sql/Main.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 4ff296f1f..b912128f9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -27,8 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; /** * Date: 2018/6/26 @@ -45,10 +43,4 @@ public static void main(String[] args) throws Exception { env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } From 87c26d71fecc1fd5aeb8d1b8c66d3077c8bcb45c Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Sun, 1 Mar 2020 14:44:42 +0800 Subject: [PATCH 007/378] =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/AsyncReqRow.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index 066708d6c..cdd4a1610 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -122,13 +122,14 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { @Override public void timeout(CRow input, ResultFuture resultFuture) throws Exception { - StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; - try { - if (null == future.get()) { - resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")); - } - } catch (Exception e) { - resultFuture.completeExceptionally(new Exception(e)); + + if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ + LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); + } + + timeOutNum ++; + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ + resultFuture.complete(null); } } From 1248591145ceffe9f59c4bd6908677283798d402 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 14:07:32 +0800 Subject: [PATCH 008/378] fix completeExceptionally desc --- core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index cdd4a1610..cd811e48f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -129,7 +129,7 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ - resultFuture.complete(null); + resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); } } From d445fcac42fbc1b03aae9d1562ea3dd64b95fe21 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 1 Mar 2020 20:47:05 +0800 Subject: [PATCH 009/378] =?UTF-8?q?=E5=8F=82=E6=95=B0=E8=B5=8B=E5=80=BC?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java | 2 +- .../java/com/dtstack/flink/sql/table/AbsSideTableParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 8c78578fb..c82489c94 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -65,7 +65,7 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; + private int asyncTimeoutNumLimit = 0; private boolean partitionedJoin = false; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 4e584cbe8..bf0b8a7a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -111,7 +111,7 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ - sideTableInfo.setAsyncTimeout(asyncTimeoutNum); + sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } } From c47553672b5f5158315b62c96fca4947dbc34856 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 2 Mar 2020 10:22:24 +0800 Subject: [PATCH 010/378] remove log level and modify objectmapper class --- core/pom.xml | 5 ----- .../src/main/java/com/dtstack/flink/sql/Main.java | 9 --------- .../com/dtstack/flink/sql/exec/ApiResult.java | 2 +- .../flink/sql/exec/ExecuteProcessHelper.java | 11 ----------- .../com/dtstack/flink/sql/exec/ParamsInfo.java | 15 ++------------- 5 files changed, 3 insertions(+), 39 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 4b5a3e67b..677f9d5ee 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -121,11 +121,6 @@ junit 4.12 - - ch.qos.logback - logback-classic - 1.1.7 - diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 4ff296f1f..5d7528869 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -27,9 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; - /** * Date: 2018/6/26 * Company: www.dtstack.com @@ -45,10 +42,4 @@ public static void main(String[] args) throws Exception { env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java index d6ca857c3..d67a39a40 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java @@ -1,6 +1,6 @@ package com.dtstack.flink.sql.exec; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2c6d74df0..d2c2926ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,8 +18,6 @@ package com.dtstack.flink.sql.exec; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.enums.ClusterMode; @@ -128,7 +126,6 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setDeployMode(deployMode) .setConfProp(confProperties) .setJarUrlList(jarURList) - .setLogLevel(logLevel) .build(); } @@ -154,8 +151,6 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); - setLogLevel(paramsInfo.getLogLevel()); - SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); @@ -348,11 +343,5 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - private static void setLogLevel(String level){ - LoggerContext loggerContext= (LoggerContext) LoggerFactory.getILoggerFactory(); - //设置全局日志级别 - ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root"); - logger.setLevel(Level.toLevel(level, Level.INFO)); - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 59249827c..27cc7702d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -38,11 +38,10 @@ public class ParamsInfo { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; - private String logLevel; private Properties confProp; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, String deployMode, String logLevel, Properties confProp) { + String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { this.sql = sql; this.name = name; this.jarUrlList = jarUrlList; @@ -50,7 +49,6 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.remoteSqlPluginPath = remoteSqlPluginPath; this.pluginLoadMode = pluginLoadMode; this.deployMode = deployMode; - this.logLevel = logLevel; this.confProp = confProp; } @@ -86,10 +84,6 @@ public Properties getConfProp() { return confProp; } - public String getLogLevel() { - return logLevel; - } - @Override public String toString() { return "ParamsInfo{" + @@ -100,7 +94,6 @@ public String toString() { ", remoteSqlPluginPath='" + remoteSqlPluginPath + '\'' + ", pluginLoadMode='" + pluginLoadMode + '\'' + ", deployMode='" + deployMode + '\'' + - ", logLevel='" + logLevel + '\'' + ", confProp=" + confProp + '}'; } @@ -160,10 +153,6 @@ public ParamsInfo.Builder setDeployMode(String deployMode) { return this; } - public ParamsInfo.Builder setLogLevel(String logLevel) { - this.logLevel = logLevel; - return this; - } public ParamsInfo.Builder setConfProp(Properties confProp) { this.confProp = confProp; @@ -172,7 +161,7 @@ public ParamsInfo.Builder setConfProp(Properties confProp) { public ParamsInfo build() { return new ParamsInfo(sql, name, jarUrlList, localSqlPluginPath, - remoteSqlPluginPath, pluginLoadMode, deployMode, logLevel, confProp); + remoteSqlPluginPath, pluginLoadMode, deployMode, confProp); } } } From fbf268e37a8dd78290641b02d159c228bccd1ea9 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 2 Mar 2020 16:28:51 +0800 Subject: [PATCH 011/378] set max batch size --- .../com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index d43104fcb..4554bb798 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -29,6 +29,8 @@ */ public class RdbTableInfo extends TargetTableInfo { + public static final int MAX_BATCH_SIZE = 10000; + public static final String URL_KEY = "url"; public static final String TABLE_NAME_KEY = "tableName"; @@ -144,6 +146,10 @@ public boolean check() { Preconditions.checkNotNull(tableName, "rdb field of tableName is required"); Preconditions.checkNotNull(userName, "rdb field of userName is required"); Preconditions.checkNotNull(password, "rdb field of password is required"); + + if (null != batchSize) { + Preconditions.checkArgument(batchSize <= MAX_BATCH_SIZE, "batchSize must be less than " + MAX_BATCH_SIZE); + } return true; } From 586c74a117b02aed822dbe311ca79419856137f3 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 4 Mar 2020 10:22:35 +0800 Subject: [PATCH 012/378] es id number check --- ...asticsearch6Side => elasticsearch6Side.md} | 0 .../table/ElasticsearchTableInfo.java | 8 +++ .../sink/elasticsearch/CustomerSinkFunc.java | 38 ++++++------ .../sink/elasticsearch/ElasticsearchSink.java | 62 +++++++------------ .../table/ElasticsearchTableInfo.java | 8 +++ pom.xml | 2 +- 6 files changed, 56 insertions(+), 62 deletions(-) rename docs/{elasticsearch6Side => elasticsearch6Side.md} (100%) diff --git a/docs/elasticsearch6Side b/docs/elasticsearch6Side.md similarity index 100% rename from docs/elasticsearch6Side rename to docs/elasticsearch6Side.md diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index ac1d712ce..b0439978e 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -23,6 +23,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.Arrays; /** * @date 2018/09/12 @@ -131,6 +135,10 @@ public boolean check() { Preconditions.checkNotNull(id, "elasticsearch type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch type of clusterName is required"); + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch type of userName is required"); Preconditions.checkNotNull(password, "elasticsearch type of password is required"); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index d6d207c9e..b1c8a5e97 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -31,9 +31,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @author yinxi @@ -42,6 +42,8 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + /** 用作ID的属性值连接符号 */ + private static final String ID_VALUE_SPLIT = "_"; private String index; @@ -57,10 +59,7 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private transient Counter outDirtyRecords; - /** 默认分隔符为'_' */ - private char sp = '_'; - - public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes){ + public CustomerSinkFunc(String index, String type, List fieldNames, List fieldTypes, List idFieldIndexes) { this.index = index; this.type = type; this.fieldNames = fieldNames; @@ -96,31 +95,30 @@ public void setOutDirtyRecords(Counter outDirtyRecords) { } private IndexRequest createIndexRequest(Row element) { + // index start at 1, + String idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } - - Map dataMap = Es6Util.rowToJsonMap(element,fieldNames,fieldTypes); + Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); - for(int i=0; i, IStreamSinkGener { - private final Logger logger = LoggerFactory.getLogger(ElasticsearchSink.class); + private final int ES_DEFAULT_PORT = 9200; + private final String ES_DEFAULT_SCHEMA = "http"; private String clusterName; @@ -103,32 +101,20 @@ public TypeInformation[] getFieldTypes() { private RichSinkFunction createEsSinkFunction() { - - Map userConfig = Maps.newHashMap(); userConfig.put("cluster.name", clusterName); // This instructs the sink to emit after every element, otherwise they would be buffered userConfig.put(org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "" + bulkFlushMaxActions); - List transports = new ArrayList<>(); - - for (String address : esAddressList) { - String[] infoArray = address.split(":"); - int port = 9200; - String host = infoArray[0]; - if (infoArray.length > 1) { - port = Integer.valueOf(infoArray[1].trim()); - } - - try { - transports.add(new HttpHost(host.trim(), port, "http")); - } catch (Exception e) { - logger.error("", e); - throw new RuntimeException(e); - } - } - CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); + List transports = esAddressList.stream() + .map(address -> address.split(":")) + .map(addressArray -> { + String host = addressArray[0].trim(); + int port = addressArray.length > 1 ? Integer.valueOf(addressArray[1].trim()) : ES_DEFAULT_PORT; + return new HttpHost(host.trim(), port, ES_DEFAULT_SCHEMA); + }).collect(Collectors.toList()); + CustomerSinkFunc customerSinkFunc = new CustomerSinkFunc(index, type, Arrays.asList(fieldNames), Arrays.asList(columnTypes), idIndexList); return new MetricElasticsearch6Sink(userConfig, transports, customerSinkFunc, esTableInfo); } @@ -151,23 +137,17 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { @Override public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { - ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; - esTableInfo = elasticsearchTableInfo; - clusterName = elasticsearchTableInfo.getClusterName(); - String address = elasticsearchTableInfo.getAddress(); - String[] addr = address.split(","); - esAddressList = Arrays.asList(addr); - index = elasticsearchTableInfo.getIndex(); - type = elasticsearchTableInfo.getEsType(); - String id = elasticsearchTableInfo.getId(); - String[] idField = StringUtils.split(id, ","); - idIndexList = new ArrayList<>(); - - for (int i = 0; i < idField.length; ++i) { - idIndexList.add(Integer.valueOf(idField[i])); + esTableInfo = (ElasticsearchTableInfo) targetTableInfo; + clusterName = esTableInfo.getClusterName(); + index = esTableInfo.getIndex(); + type = esTableInfo.getEsType(); + columnTypes = esTableInfo.getFieldTypes(); + esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); + + String id = esTableInfo.getId(); + if (!StringUtils.isEmpty(id)) { + idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); } - - columnTypes = elasticsearchTableInfo.getFieldTypes(); return this; } } diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index b0156b660..5709d609c 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -20,6 +20,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.Arrays; /** * @author yinxi @@ -127,6 +131,10 @@ public boolean check() { Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); + Arrays.stream(StringUtils.split(id, ",")).forEach(number ->{ + Preconditions.checkArgument(NumberUtils.isNumber(number),"id must be a numeric type"); + }); + if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); Preconditions.checkNotNull(password, "elasticsearch6 type of password is required"); diff --git a/pom.xml b/pom.xml index 2794f9004..5493645e9 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ mysql hbase elasticsearch5 + elasticsearch6 mongo redis5 launcher @@ -33,7 +34,6 @@ impala db2 polardb - elasticsearch6 From 284d52d4ecb5dbcf51a1b379731f9b664586d183 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 4 Mar 2020 13:23:54 +0800 Subject: [PATCH 013/378] id allow as null --- .../sink/elasticsearch/CustomerSinkFunc.java | 27 +++++++++++-------- .../table/ElasticsearchTableInfo.java | 9 ++++--- .../sink/elasticsearch/ElasticsearchSink.java | 10 +------ .../table/ElasticsearchTableInfo.java | 9 ++++--- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index a95d477de..1f3efb8d7 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -47,6 +48,8 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); + private static final String ID_VALUE_SPLIT = "_"; + private String index; private String type; @@ -93,15 +96,11 @@ public void setOutRecords(Counter outRecords) { } private IndexRequest createIndexRequest(Row element) { - - List idFieldList = new ArrayList<>(); - for(int index : idFieldIndexList){ - if(index >= element.getArity()){ - continue; - } - - idFieldList.add(element.getField(index).toString()); - } + // index start at 1, + String idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); @@ -109,11 +108,17 @@ private IndexRequest createIndexRequest(Row element) { dataMap.put(fieldNames.get(i), element.getField(i)); } - String id = StringUtils.join(idFieldList, sp); + if (StringUtils.isEmpty(idFieldStr)) { + return Requests.indexRequest() + .index(index) + .type(type) + .source(dataMap); + } + return Requests.indexRequest() .index(index) .type(type) - .id(id) + .id(idFieldStr) .source(dataMap); } } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index b0439978e..9681742c2 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -132,12 +132,13 @@ public boolean check() { Preconditions.checkNotNull(address, "elasticsearch type of address is required"); Preconditions.checkNotNull(index, "elasticsearch type of index is required"); Preconditions.checkNotNull(esType, "elasticsearch type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch type of clusterName is required"); - Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { - Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); - }); + if (!StringUtils.isEmpty(id)) { + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + } if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch type of userName is required"); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index cc650f670..2646c50e9 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -127,14 +127,6 @@ public void emitDataStream(DataStream> dataStream) { } } - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public void setBulkFlushMaxActions(int bulkFlushMaxActions) { - this.bulkFlushMaxActions = bulkFlushMaxActions; - } - @Override public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; @@ -143,8 +135,8 @@ public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { type = esTableInfo.getEsType(); columnTypes = esTableInfo.getFieldTypes(); esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); - String id = esTableInfo.getId(); + if (!StringUtils.isEmpty(id)) { idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); } diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 5709d609c..3cc3dd9ff 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -128,12 +128,13 @@ public boolean check() { Preconditions.checkNotNull(address, "elasticsearch6 type of address is required"); Preconditions.checkNotNull(index, "elasticsearch6 type of index is required"); Preconditions.checkNotNull(esType, "elasticsearch6 type of type is required"); - Preconditions.checkNotNull(id, "elasticsearch6 type of id is required"); Preconditions.checkNotNull(clusterName, "elasticsearch6 type of clusterName is required"); - Arrays.stream(StringUtils.split(id, ",")).forEach(number ->{ - Preconditions.checkArgument(NumberUtils.isNumber(number),"id must be a numeric type"); - }); + if (!StringUtils.isEmpty(id)) { + Arrays.stream(StringUtils.split(id, ",")).forEach(number -> { + Preconditions.checkArgument(NumberUtils.isNumber(number), "id must be a numeric type"); + }); + } if (isAuthMesh()) { Preconditions.checkNotNull(userName, "elasticsearch6 type of userName is required"); From 126a29564b43c6fafca7b0236e1e83228e42cccc Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 16:14:19 +0800 Subject: [PATCH 014/378] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/StreamEnvConfigManager.java | 18 ++++++----- .../flink/sql/option/OptionParser.java | 4 +-- .../flink/sql/parser/CreateFuncParser.java | 8 ++--- .../flink/sql/side/SidePredicatesParser.java | 2 ++ .../dtstack/flink/sql/side/SideSQLParser.java | 4 +++ .../dtstack/flink/sql/side/SideSqlExec.java | 22 +++++++------ .../flink/sql/table/AbsSideTableParser.java | 2 +- .../com/dtstack/flink/sql/util/ClassUtil.java | 3 +- .../com/dtstack/flink/sql/util/DateUtil.java | 32 +++++++++---------- .../com/dtstack/flink/sql/util/JDBCUtils.java | 4 +-- 10 files changed, 56 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index ab74edabc..b453c9414 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -258,6 +258,8 @@ private static Optional createStateBackend(String backendType, Str checkpointDataUriEmptyCheck(checkpointDataUri, backendType); stateBackend = new RocksDBStateBackend(checkpointDataUri, BooleanUtils.toBoolean(backendIncremental)); break; + default: + break; } return stateBackend == null ? Optional.empty() : Optional.of(stateBackend); } @@ -317,14 +319,14 @@ private static void verityTtl(String ttlMintimeStr, String ttlMaxtimeStr) { * @return */ private static Long getTtlTime(Integer timeNumber, String timeUnit) { - if (timeUnit.equalsIgnoreCase("d")) { - return timeNumber * 1000l * 60 * 60 * 24; - } else if (timeUnit.equalsIgnoreCase("h")) { - return timeNumber * 1000l * 60 * 60; - } else if (timeUnit.equalsIgnoreCase("m")) { - return timeNumber * 1000l * 60; - } else if (timeUnit.equalsIgnoreCase("s")) { - return timeNumber * 1000l; + if ("d".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60 * 24; + } else if ("h".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60; + } else if ("m".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60; + } else if ("s".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L; } else { throw new RuntimeException("not support " + timeNumber + timeUnit); } diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 43f599d14..0e8bda1fe 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -114,7 +114,7 @@ public List getProgramExeArgList() throws Exception { } public static void main(String[] args) throws Exception { - OptionParser OptionParser = new OptionParser(args); - System.out.println(OptionParser.getOptions()); + OptionParser optionParser = new OptionParser(args); + System.out.println(optionParser.getOptions()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index 670d98a7e..fc6a7e16c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -32,18 +32,18 @@ public class CreateFuncParser implements IParser { - private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String FUNC_PATTERN_STR = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; - private static final Pattern funcPattern = Pattern.compile(funcPatternStr); + private static final Pattern FUNC_PATTERN = Pattern.compile(FUNC_PATTERN_STR); @Override public boolean verify(String sql) { - return funcPattern.matcher(sql).find(); + return FUNC_PATTERN.matcher(sql).find(); } @Override public void parseSql(String sql, SqlTree sqlTree) { - Matcher matcher = funcPattern.matcher(sql); + Matcher matcher = FUNC_PATTERN.matcher(sql); if(matcher.find()){ String type = matcher.group(1); String funcName = matcher.group(2); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 50103a9f5..64f9e646b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -101,6 +101,8 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; + default: + break; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index c12bc981e..b7d886566 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -131,6 +131,8 @@ private void checkAndReplaceMultiJoin(SqlNode sqlNode, Set sideTableSet) checkAndReplaceMultiJoin(unionLeft, sideTableSet); checkAndReplaceMultiJoin(unionRight, sideTableSet); break; + default: + break; } } @@ -204,6 +206,8 @@ private Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList); + default: + break; } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index d54c87c17..1689949ca 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -90,7 +90,7 @@ public class SideSqlExec { private String tmpFields = null; - private SideSQLParser sideSQLParser = new SideSQLParser(); + private SideSQLParser sideSqlParser = new SideSQLParser(); private SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); private Map localTableCache = Maps.newHashMap(); @@ -109,8 +109,8 @@ public void exec(String sql, Map sideTableMap, StreamTabl LOG.error("fill predicates for sideTable fail ", e); } - sideSQLParser.setLocalTableCache(localTableCache); - Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); + sideSqlParser.setLocalTableCache(localTableCache); + Queue exeQueue = sideSqlParser.getExeQueue(sql, sideTableMap.keySet()); Object pollObj = null; //need clean @@ -140,7 +140,7 @@ public void exec(String sql, Map sideTableMap, StreamTabl LOG.info("exec sql: " + pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ - AliasInfo aliasInfo = parseASNode(pollSqlNode); + AliasInfo aliasInfo = parseAsNode(pollSqlNode); Table table = tableEnv.sqlQuery(aliasInfo.getName()); tableEnv.registerTable(aliasInfo.getAlias(), table); localTableCache.put(aliasInfo.getAlias(), table); @@ -151,9 +151,9 @@ public void exec(String sql, Map sideTableMap, StreamTabl } } else if (pollSqlNode.getKind() == WITH_ITEM) { SqlWithItem sqlWithItem = (SqlWithItem) pollSqlNode; - String TableAlias = sqlWithItem.name.toString(); + String tableAlias = sqlWithItem.name.toString(); Table table = tableEnv.sqlQuery(sqlWithItem.query.toString()); - tableEnv.registerTable(TableAlias, table); + tableEnv.registerTable(tableAlias, table); } }else if (pollObj instanceof JoinInfo){ @@ -253,11 +253,13 @@ private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, H } } break; + default: + break; } } - public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { + public AliasInfo parseAsNode(SqlNode sqlNode) throws SqlParseException { SqlKind sqlKind = sqlNode.getKind(); if(sqlKind != AS){ throw new RuntimeException(sqlNode + " is not 'as' operator"); @@ -487,6 +489,8 @@ public SqlNode filterNodeWithTargetName(SqlNode sqlNode, String targetTableName) }else{ return null; } + default: + break; } return null; @@ -709,7 +713,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } localTableCache.putAll(tableCache); - Queue exeQueue = sideSQLParser.getExeQueue(result.getExecSql(), sideTableMap.keySet()); + Queue exeQueue = sideSqlParser.getExeQueue(result.getExecSql(), sideTableMap.keySet()); Object pollObj = null; //need clean @@ -759,7 +763,7 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSqlNode) throws SqlParseException { - AliasInfo aliasInfo = parseASNode(pollSqlNode); + AliasInfo aliasInfo = parseAsNode(pollSqlNode); if (localTableCache.containsKey(aliasInfo.getName())) { return; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 520ca2138..49c5629bc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -87,7 +87,7 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ + if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index feebbe3f1..4bb4ff0d8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -86,7 +86,8 @@ public static Class stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - + default: + break; } throw new RuntimeException("不支持 " + str + " 类型"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 5bfa2f203..da5a42b57 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -116,9 +116,9 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if(scope.equals("MS")){ + if("MS".equals(scope)){ return getTodayStart(day)*1000; - }else if(scope.equals("S")){ + }else if("S".equals(scope)){ return getTodayStart(day); }else{ return getTodayStart(day); @@ -154,9 +154,9 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if(scope.equals("MS")){ + if("MS".equals(scope)){ return getNextDayStart(day)*1000; - }else if(scope.equals("S")){ + }else if("S".equals(scope)){ return getNextDayStart(day); }else{ return getNextDayStart(day); @@ -335,7 +335,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -344,7 +344,7 @@ public static String getDateStrTOFormat(String day, String inFormat, String outF return dayBefore; } - public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { + public static long getDateMillToFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -470,11 +470,11 @@ public static long getMillByDay(int severalDays,String condition) { if(condition==null){ return getMillToDay(cal,dateT); } - if(condition.equals("-")){ + if("-".equals(condition)){ dateT = (cal.get(Calendar.DATE) - severalDays); return getMillToDay(cal,dateT); } - if(condition.equals("+")){ + if("+".equals(condition)){ dateT = (cal.get(Calendar.DATE) + severalDays); return getMillToDay(cal,dateT); } @@ -490,11 +490,11 @@ public static long getStampByDay(int severalDays,String condition) { if(condition==null){ return getStampToDay(cal,dateT); } - if(condition.equals("-")){ + if("-".equals(condition)){ dateT = (cal.get(Calendar.DATE) - severalDays); return getStampToDay(cal,dateT); } - if(condition.equals("+")){ + if("+".equals(condition)){ dateT = (cal.get(Calendar.DATE) + severalDays); return getStampToDay(cal,dateT); } @@ -575,8 +575,8 @@ public static String getDate(Date date, String format) { */ public static long stringToLong(String day, String format) throws ParseException { SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long Date = dateFormat.parse(day).getTime(); - return Date; + long date = dateFormat.parse(day).getTime(); + return date; } /** @@ -588,8 +588,8 @@ public static long stringToLong(String day, String format) throws ParseException public static Date stringToDate(String day, String format) { try { SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date Date = dateFormat.parse(day); - return Date; + Date date = dateFormat.parse(day); + return date; } catch (ParseException e) { return new Date(); } @@ -608,8 +608,8 @@ public static String longToString(long day, String format) throws ParseException day=day*1000; } SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String Date = dateFormat.format(day); - return Date; + String date = dateFormat.format(day); + return date; } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index fde2f166e..3d66d8a6d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -28,10 +28,10 @@ public class JDBCUtils { private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - public final static String lock_str = "jdbc_lock_str"; + public final static String LOCK_STR = "jdbc_lock_str"; public static void forName(String clazz, ClassLoader classLoader) { - synchronized (lock_str){ + synchronized (LOCK_STR){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); From e0f289be82541f7f630e91f9506f8be08ef63ae4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 16:47:04 +0800 Subject: [PATCH 015/378] codeview fix --- .../flink/sql/sink/console/table/TablePrintUtil.java | 12 +++++++++--- .../flink/sql/launcher/ClusterClientFactory.java | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 8813da619..39abb81b4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -58,7 +58,9 @@ public static TablePrintUtil build(String[][] data) { public static TablePrintUtil build(List data) { TablePrintUtil self = new TablePrintUtil(); self.data = new ArrayList<>(); - if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); + if (data.size() <= 0) { + throw new RuntimeException("数据源至少得有一行吧"); + } Object obj = data.get(0); @@ -70,7 +72,9 @@ public static TablePrintUtil build(List data) { int length = ((List) obj).size(); for (Object item : data) { List col = (List) item; - if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); + if (col.size() != length) { + throw new RuntimeException("数据源每列长度必须一致"); + } self.data.add(col.toArray(new String[length])); } } else { @@ -163,7 +167,9 @@ private int[] getColLengths() { if (equilong) {//如果等宽表格 int max = 0; for (int len : result) { - if (len > max) max = len; + if (len > max) { + max = len; + } } for (int i = 0; i < result.length; i++) { result[i] = max; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 14cd847b5..7f4ded520 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -89,7 +89,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); + config.setString("fs.hdfs.hadoopconf", yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -166,7 +166,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { + if (!"application".equals(it.next())) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { From e51f1cfdf5ccf41410998242c8f78a22d0875028 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 17:04:12 +0800 Subject: [PATCH 016/378] codereview --- .../flink/sql/side/cassandra/table/CassandraSideParser.java | 3 +++ .../flink/sql/side/clickhouse/ClickhouseAllReqRow.java | 6 +++--- .../flink/sql/sink/console/table/TablePrintUtil.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 62dd753b9..d2b93563b 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -99,6 +99,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map getColList(Object obj) { Method[] methods = obj.getClass().getMethods(); for (Method m : methods) { StringBuilder getMethodName = new StringBuilder(m.getName()); - if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { + if ("get".equals(getMethodName.substring(0, 3)) && !"getClass".equals(m.getName())) { Col col = new Col(); col.getMethodName = getMethodName.toString(); char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); From 841c1fe931d0e1bdb838b4eea9034123a5b61e81 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Wed, 4 Mar 2020 18:12:19 +0800 Subject: [PATCH 017/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index d2c2926ef..5cd1248e2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -188,17 +188,23 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. return jarUrlList; } - public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + private static void sqlTranslation(String localSqlPluginPath, + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); } for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { if (LOG.isInfoEnabled()) { LOG.info("exe-sql:\n" + result.getExecSql()); } + boolean isSide = false; for (String tableName : result.getTargetTableList()) { if (sqlTree.getTmpTableMap().containsKey(tableName)) { @@ -208,7 +214,7 @@ public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironm SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); - sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); + sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); } else { for (String sourceTable : result.getSourceTableList()) { if (sideTableMap.containsKey(sourceTable)) { @@ -218,10 +224,14 @@ public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironm } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { + System.out.println("----------exec sql without dimension join-----------"); + System.out.println("----------real sql exec is--------------------------"); + System.out.println(result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { + System.out.println(); LOG.info("exec sql: " + result.getExecSql()); } } From f56801c3180bef5347adced6a352dd6e190c7f2d Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 4 Mar 2020 22:41:53 +0800 Subject: [PATCH 018/378] code review --- .../side/cassandra/CassandraAllReqRow.java | 10 ++- .../side/cassandra/CassandraAllSideInfo.java | 10 +-- .../side/cassandra/CassandraAsyncReqRow.java | 9 ++- .../cassandra/CassandraAsyncSideInfo.java | 10 +-- .../cassandra/table/CassandraSideParser.java | 14 ++--- .../table/CassandraSideTableInfo.java | 4 +- .../sink/cassandra/CassandraOutputFormat.java | 4 +- .../sql/sink/cassandra/CassandraSink.java | 4 +- .../cassandra/table/CassandraSinkParser.java | 10 +-- .../cassandra/table/CassandraTableInfo.java | 4 +- .../side/clickhouse/ClickhouseAllReqRow.java | 4 +- .../clickhouse/ClickhouseAllSideInfo.java | 4 +- .../clickhouse/ClickhouseAsyncReqRow.java | 4 +- .../clickhouse/ClickhouseAsyncSideInfo.java | 4 +- .../table/ClickhouseSideParser.java | 6 +- .../table/ClickhouseSinkParser.java | 6 +- .../sql/sink/console/ConsoleOutputFormat.java | 4 +- .../flink/sql/sink/console/ConsoleSink.java | 4 +- .../sink/console/table/ConsoleSinkParser.java | 10 +-- .../sink/console/table/ConsoleTableInfo.java | 4 +- .../flink/sql/constrant/ConfigConstrant.java | 6 +- .../dtstack/flink/sql/enums/ColumnType.java | 63 +++++++++++++++++-- .../flink/sql/enums/ECacheContentType.java | 9 +++ .../dtstack/flink/sql/enums/ECacheType.java | 13 +++- .../flink/sql/enums/EDatabaseType.java | 9 +++ .../flink/sql/enums/EPluginLoadMode.java | 6 ++ .../flink/sql/enums/EStateBackend.java | 9 +++ .../environment/StreamEnvConfigManager.java | 6 +- .../flink/sql/exec/ExecuteProcessHelper.java | 36 +++++------ .../DtNestRowDeserializationSchema.java | 8 +-- .../flink/sql/option/OptionParser.java | 2 +- ...t.java => AbstractDtRichOutputFormat.java} | 2 +- .../dtstack/flink/sql/parser/SqlParser.java | 12 ++-- .../com/dtstack/flink/sql/parser/SqlTree.java | 8 +-- ...leInfo.java => AbstractSideTableInfo.java} | 4 +- .../{AllReqRow.java => BaseAllReqRow.java} | 8 +-- ...{AsyncReqRow.java => BaseAsyncReqRow.java} | 14 ++--- .../side/{SideInfo.java => BaseSideInfo.java} | 22 +++---- .../flink/sql/side/SidePredicatesParser.java | 7 +-- .../dtstack/flink/sql/side/SideSqlExec.java | 14 ++--- .../flink/sql/side/StreamSideFactory.java | 10 +-- ...sSideCache.java => AbstractSideCache.java} | 8 +-- .../flink/sql/side/cache/LRUSideCache.java | 6 +- .../sql/side/operator/SideAsyncOperator.java | 16 ++--- .../operator/SideWithAllCacheOperator.java | 18 +++--- .../flink/sql/sink/IStreamSinkGener.java | 4 +- .../flink/sql/sink/StreamSinkFactory.java | 12 ++-- .../flink/sql/source/IStreamSourceGener.java | 4 +- .../flink/sql/source/StreamSourceFactory.java | 12 ++-- ...rser.java => AbstractSideTableParser.java} | 38 +++++------ ...eParser.java => AbstractSourceParser.java} | 16 ++--- ...Info.java => AbstractSourceTableInfo.java} | 2 +- ...{TableInfo.java => AbstractTableInfo.java} | 2 +- ...rser.java => AbstractTableInfoParser.java} | 18 +++--- ...leParser.java => AbstractTableParser.java} | 16 ++--- ...Info.java => AbstractTargetTableInfo.java} | 2 +- .../sql/table/ITableFieldDealHandler.java | 2 +- .../com/dtstack/flink/sql/util/DateUtil.java | 1 - .../dtstack/flink/sql/util/PluginUtil.java | 10 +-- ....java => AbstractCustomerWaterMarker.java} | 4 +- .../CustomerWaterMarkerForLong.java | 2 +- .../CustomerWaterMarkerForTimeStamp.java | 3 +- .../sql/watermarker/WaterMarkerAssigner.java | 8 +-- .../sql/side/SidePredicatesParserTest.java | 4 +- .../flink/sql/side/db2/Db2AllReqRow.java | 8 +-- .../flink/sql/side/db2/Db2AllSideInfo.java | 4 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 4 +- .../flink/sql/side/db2/Db2AsyncSideInfo.java | 4 +- .../sql/side/db2/table/Db2SideParser.java | 6 +- .../flink/sql/sink/db/table/DbSinkParser.java | 6 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +-- .../table/ElasticsearchTableInfo.java | 4 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 8 +-- .../sql/side/hbase/HbaseAllSideInfo.java | 10 +-- .../sql/side/hbase/HbaseAsyncReqRow.java | 14 ++--- .../sql/side/hbase/HbaseAsyncSideInfo.java | 10 +-- ...ler.java => AbstractRowKeyModeDealer.java} | 12 ++-- .../PreRowKeyModeDealerDealer.java | 9 ++- .../rowkeydealer/RowKeyEqualModeDealer.java | 6 +- ...ator.java => AbstractReplaceOperator.java} | 4 +- .../sql/side/hbase/Md5ReplaceOperator.java | 2 +- .../sql/side/hbase/enums/EReplaceOpType.java | 9 ++- .../sql/side/hbase/enums/EReplaceType.java | 6 ++ .../sql/side/hbase/table/HbaseSideParser.java | 12 ++-- .../side/hbase/table/HbaseSideTableInfo.java | 4 +- .../sql/side/hbase/utils/HbaseUtils.java | 2 + .../sql/sink/hbase/HbaseOutputFormat.java | 4 +- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 10 +-- .../sql/sink/hbase/table/HbaseTableInfo.java | 4 +- hbase/pom.xml | 7 +++ .../sql/side/impala/ImpalaAllReqRow.java | 6 +- .../sql/side/impala/ImpalaAllSideInfo.java | 7 +-- .../sql/side/impala/ImpalaAsyncReqRow.java | 4 +- .../sql/side/impala/ImpalaAsyncSideInfo.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 7 ++- .../flink/sql/sink/impala/ImpalaSink.java | 4 +- .../sink/impala/table/ImpalaSinkParser.java | 7 ++- .../sql/sink/kafka/table/KafkaSinkParser.java | 8 +-- .../sink/kafka/table/KafkaSinkTableInfo.java | 5 +- .../source/kafka/table/KafkaSourceParser.java | 10 +-- .../kafka/table/KafkaSourceTableInfo.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/sink/kafka/KafkaSink.java | 4 +- .../flink/sql/source/kafka/KafkaSource.java | 6 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 8 +-- .../flink/sql/side/kudu/KuduAllSideInfo.java | 10 +-- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 4 +- .../sql/side/kudu/KuduAsyncSideInfo.java | 10 +-- .../sql/side/kudu/table/KuduSideParser.java | 10 +-- .../side/kudu/table/KuduSideTableInfo.java | 4 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 4 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 4 +- .../sql/sink/kudu/table/KuduSinkParser.java | 10 +-- .../sql/sink/kudu/table/KuduTableInfo.java | 4 +- .../flink/sql/side/mongo/MongoAllReqRow.java | 8 +-- .../sql/side/mongo/MongoAllSideInfo.java | 10 +-- .../sql/side/mongo/MongoAsyncReqRow.java | 14 ++--- .../sql/side/mongo/MongoAsyncSideInfo.java | 13 ++-- .../sql/side/mongo/table/MongoSideParser.java | 12 ++-- .../side/mongo/table/MongoSideTableInfo.java | 4 +- .../sql/sink/mongo/MongoOutputFormat.java | 4 +- .../flink/sql/sink/mongo/MongoSink.java | 4 +- .../sql/sink/mongo/table/MongoSinkParser.java | 10 +-- .../sql/sink/mongo/table/MongoTableInfo.java | 4 +- .../flink/sql/side/mysql/MysqlAllReqRow.java | 4 +- .../sql/side/mysql/MysqlAllSideInfo.java | 4 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 4 +- .../sql/side/mysql/MysqlAsyncSideInfo.java | 4 +- .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 6 +- .../sql/side/oracle/OracleAllReqRow.java | 4 +- .../sql/side/oracle/OracleAllSideInfo.java | 5 +- .../sql/side/oracle/OracleAsyncReqRow.java | 4 +- .../sql/side/oracle/OracleAsyncSideInfo.java | 8 +-- .../side/oracle/table/OracleSideParser.java | 6 +- .../sink/oracle/table/OracleSinkParser.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 4 +- .../sql/side/polardb/PolardbAllSideInfo.java | 4 +- .../sql/side/polardb/PolardbAsyncReqRow.java | 4 +- .../side/polardb/PolardbAsyncSideInfo.java | 4 +- .../side/polardb/table/PolardbSideParser.java | 6 +- .../sink/polardb/table/PolardbSinkParser.java | 6 +- .../side/postgresql/PostgresqlAllReqRow.java | 4 +- .../postgresql/PostgresqlAllSideInfo.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 4 +- .../postgresql/PostgresqlAsyncSideInfo.java | 4 +- .../table/PostgresqlSideParser.java | 6 +- .../table/PostgresqlSinkParser.java | 6 +- .../side/rdb/all/AbstractRdbAllReqRow.java | 8 +-- .../sql/side/rdb/all/RdbAllSideInfo.java | 10 +-- .../sql/side/rdb/async/RdbAsyncReqRow.java | 8 +-- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 11 ++-- .../sql/side/rdb/table/RdbSideParser.java | 8 +-- .../sql/side/rdb/table/RdbSideTableInfo.java | 4 +- .../flink/sql/sink/rdb/AbstractRdbSink.java | 4 +- .../rdb/format/AbstractJDBCOutputFormat.java | 4 +- .../sql/sink/rdb/table/RdbSinkParser.java | 9 ++- .../sql/sink/rdb/table/RdbTableInfo.java | 6 +- .../sink/rdb/writer/AbstractUpsertWriter.java | 12 ++-- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 6 +- .../flink/sql/side/redis/RedisAllReqRow.java | 4 +- .../sql/side/redis/RedisAllSideInfo.java | 10 +-- .../sql/side/redis/RedisAsyncReqRow.java | 4 +- .../sql/side/redis/RedisAsyncSideInfo.java | 10 +-- .../sql/side/redis/table/RedisSideParser.java | 8 +-- .../sql/side/redis/table/RedisSideReqRow.java | 8 +-- .../side/redis/table/RedisSideTableInfo.java | 4 +- .../sql/sink/redis/RedisOutputFormat.java | 4 +- .../flink/sql/sink/redis/RedisSink.java | 4 +- .../sql/sink/redis/table/RedisSinkParser.java | 8 +-- .../sql/sink/redis/table/RedisTableInfo.java | 4 +- .../CustomerSocketTextStreamFunction.java | 4 +- .../serversocket/ServersocketSource.java | 4 +- .../table/ServersocketSourceParser.java | 8 +-- .../table/ServersocketSourceTableInfo.java | 4 +- .../side/sqlserver/SqlserverAllReqRow.java | 4 +- .../side/sqlserver/SqlserverAllSideInfo.java | 4 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 2 +- .../sqlserver/SqlserverAsyncSideInfo.java | 4 +- .../sqlserver/table/SqlserverSideParser.java | 6 +- .../sqlserver/table/SqlserverSinkParser.java | 6 +- 188 files changed, 731 insertions(+), 640 deletions(-) rename core/src/main/java/com/dtstack/flink/sql/outputformat/{DtRichOutputFormat.java => AbstractDtRichOutputFormat.java} (95%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideTableInfo.java => AbstractSideTableInfo.java} (96%) rename core/src/main/java/com/dtstack/flink/sql/side/{AllReqRow.java => BaseAllReqRow.java} (88%) rename core/src/main/java/com/dtstack/flink/sql/side/{AsyncReqRow.java => BaseAsyncReqRow.java} (91%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideInfo.java => BaseSideInfo.java} (91%) rename core/src/main/java/com/dtstack/flink/sql/side/cache/{AbsSideCache.java => AbstractSideCache.java} (84%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSideTableParser.java => AbstractSideTableParser.java} (71%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSourceParser.java => AbstractSourceParser.java} (81%) rename core/src/main/java/com/dtstack/flink/sql/table/{SourceTableInfo.java => AbstractSourceTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfo.java => AbstractTableInfo.java} (98%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfoParser.java => AbstractTableInfoParser.java} (85%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsTableParser.java => AbstractTableParser.java} (90%) rename core/src/main/java/com/dtstack/flink/sql/table/{TargetTableInfo.java => AbstractTargetTableInfo.java} (94%) rename core/src/main/java/com/dtstack/flink/sql/watermarker/{AbsCustomerWaterMarker.java => AbstractCustomerWaterMarker.java} (94%) rename hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/{AbsRowKeyModeDealer.java => AbstractRowKeyModeDealer.java} (87%) rename hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/{ReplaceOperator.java => AbstractReplaceOperator.java} (93%) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index e30b6dfc8..6d1953bfc 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -28,10 +28,10 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; @@ -60,14 +60,12 @@ * * @author xuqianjin */ -public class CassandraAllReqRow extends AllReqRow { +public class CassandraAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 54015343561288219L; private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class); - private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver"; - private static final int CONN_RETRY_NUM = 3; private static final int FETCH_SIZE = 1000; @@ -77,7 +75,7 @@ public class CassandraAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index fa665f9a0..5d95dc9bf 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -37,16 +37,16 @@ * * @author xuqianjin */ -public class CassandraAllSideInfo extends SideInfo { +public class CassandraAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -8690814317653033557L; - public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; sqlCondition = "select ${selectField} from ${tableName} "; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index a60116e34..3020a73d3 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -30,11 +30,11 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; @@ -56,7 +56,6 @@ import java.net.InetAddress; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -66,7 +65,7 @@ * * @author xuqianjin */ -public class CassandraAsyncReqRow extends AsyncReqRow { +public class CassandraAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 6631584128079864735L; @@ -82,7 +81,7 @@ public class CassandraAsyncReqRow extends AsyncReqRow { private transient ListenableFuture session; private transient CassandraSideTableInfo cassandraSideTableInfo; - public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 3557f0f73..82055b94c 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -39,16 +39,16 @@ * * @author xuqianjin */ -public class CassandraAsyncSideInfo extends SideInfo { +public class CassandraAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4403313049809013362L; - public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index d2b93563b..e893e56b0 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -19,18 +19,16 @@ package com.dtstack.flink.sql.side.cassandra.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; -import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -38,7 +36,7 @@ * * @author xuqianjin */ -public class CassandraSideParser extends AbsSideTableParser { +public class CassandraSideParser extends AbstractSideTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; @@ -73,7 +71,7 @@ public CassandraSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo(); cassandraSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraSideTableInfo); @@ -96,7 +94,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class CassandraOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index eb7b23b53..26152a7d3 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.cassandra.table.CassandraTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,7 +63,7 @@ public CassandraSink() { } @Override - public CassandraSink genStreamSink(TargetTableInfo targetTableInfo) { + public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { CassandraTableInfo cassandraTableInfo = (CassandraTableInfo) targetTableInfo; this.address = cassandraTableInfo.getAddress(); this.tableName = cassandraTableInfo.getTableName(); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java index 4c68e71ae..9ef8639ba 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +33,7 @@ * * @author xuqianjin */ -public class CassandraSinkParser extends AbsTableParser { +public class CassandraSinkParser extends AbstractTableParser { public static final String ADDRESS_KEY = "address"; @@ -60,7 +60,7 @@ public class CassandraSinkParser extends AbsTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { CassandraTableInfo cassandraTableInfo = new CassandraTableInfo(); cassandraTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraTableInfo); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index c6626c42a..ffb5fa876 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class CassandraTableInfo extends TargetTableInfo { +public class CassandraTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "cassandra"; diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index bd3d033ff..c9a0c447b 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java index 973c069b9..43fbeaa56 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -28,7 +28,7 @@ public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 3733d7350..e6f008d63 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -37,7 +37,7 @@ public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java index 254561de0..eec5fbe74 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -29,7 +29,7 @@ public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java index 7be387fd8..ab285c37f 100644 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.clickhouse.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -38,8 +38,8 @@ public class ClickhouseSideParser extends RdbSideParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 8c3df93d7..5b0f2598f 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -30,8 +30,8 @@ public class ClickhouseSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index a397036ef..e99c76d03 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends DtRichOutputFormat { +public class ConsoleOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java index 77a3efea2..6cfdd2d3d 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.console; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -80,7 +80,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { + public ConsoleSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { return this; } } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java index e77444bfd..93ed02420 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java @@ -18,13 +18,13 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -32,9 +32,9 @@ * * @author xuqianjin */ -public class ConsoleSinkParser extends AbsTableParser { +public class ConsoleSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); consoleTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, consoleTableInfo); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java index 4b286c667..6fd7063c6 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -26,7 +26,7 @@ * * @author xuqianjin */ -public class ConsoleTableInfo extends TargetTableInfo { +public class ConsoleTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "console"; diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index dcda4c093..d5b13ce12 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -65,10 +65,10 @@ public class ConfigConstrant { // restart plocy - public static final int failureRate = 3; + public static final int FAILUEE_RATE = 3; - public static final int failureInterval = 6; //min + public static final int FAILUEE_INTERVAL = 6; //min - public static final int delayInterval = 10; //sec + public static final int DELAY_INTERVAL = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 749bbc907..7f3f0019c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -46,11 +46,66 @@ * @author huyifan.zju@163.com */ public enum ColumnType { - STRING, VARCHAR, CHAR, - INT, MEDIUMINT, TINYINT, DATETIME, SMALLINT, BIGINT, - DOUBLE, FLOAT, + /* + * string + */ + STRING, + /** + * varchar + */ + VARCHAR, + /** + * char + */ + CHAR, + /** + * int + */ + INT, + /** + * mediumint + */ + MEDIUMINT, + /** + * tinyint + */ + TINYINT, + /** + * datetime + */ + DATETIME, + /** + * smallint + */ + SMALLINT, + /** + * bigint + */ + BIGINT, + /** + * double + */ + DOUBLE, + /** + * float + */ + FLOAT, + /** + * boolean + */ BOOLEAN, - DATE, TIMESTAMP, DECIMAL; + /** + * date + */ + DATE, + /** + * timestamp + */ + TIMESTAMP, + /** + * decimal + */ + DECIMAL; public static ColumnType fromString(String type) { if(type == null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java index 66160d820..49e352757 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java @@ -30,8 +30,17 @@ public enum ECacheContentType { + /** + * 无 + */ MissVal(0), + /** + * 1行 + */ SingleLine(1), + /** + * 多行 + */ MultiLine(2); int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java index 582148c2c..9d5bb5d11 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java @@ -27,7 +27,18 @@ * @author xuchao */ public enum ECacheType { - NONE, LRU, ALL; + /** + * none + */ + NONE, + /** + * lru + */ + LRU, + /** + * all + */ + ALL; public static boolean isValid(String type){ for(ECacheType tmpType : ECacheType.values()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java index 9b01bf052..7d2235f2e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java @@ -26,8 +26,17 @@ */ public enum EDatabaseType { + /** + * mysql + */ MYSQL, + /** + * sqlserver + */ SQLSERVER, + /** + * oracle + */ ORACLE, } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java index 6cb027ac3..439966dd2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java @@ -26,7 +26,13 @@ */ public enum EPluginLoadMode { + /** + * 0:classpath + */ CLASSPATH(0), + /** + * 1:shipfile + */ SHIPFILE(1); private int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java index a8f926175..098cb57fe 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java @@ -25,8 +25,17 @@ * @author maqi */ public enum EStateBackend { + /** + * memory + */ MEMORY, + /** + * rockdb + */ ROCKSDB, + /** + * filesystem + */ FILESYSTEM; public static EStateBackend convertFromString(String type) { diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index b453c9414..9db50e65a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -102,9 +102,9 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s }); streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + ConfigConstrant.FAILUEE_RATE, + Time.of(ConfigConstrant.FAILUEE_INTERVAL, TimeUnit.MINUTES), + Time.of(ConfigConstrant.DELAY_INTERVAL, TimeUnit.SECONDS) )); // checkpoint config diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2c6d74df0..ed54a9c48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -36,12 +36,12 @@ import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.parser.SqlTree; import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; @@ -159,7 +159,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); - Map sideTableMap = Maps.newHashMap(); + Map sideTableMap = Maps.newHashMap(); Map registerTableCache = Maps.newHashMap(); //register udf @@ -193,7 +193,7 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. return jarUrlList; } - public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { + public static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv, SqlTree sqlTree, Map sideTableMap, Map registerTableCache, StreamQueryConfig queryConfig) throws Exception { SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { @@ -264,14 +264,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @throws Exception */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { Set pluginClassPatshSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - if (tableInfo instanceof SourceTableInfo) { + if (tableInfo instanceof AbstractSourceTableInfo) { - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); //Note --- parameter conversion function can not be used inside a function of the type of polymerization @@ -302,21 +302,21 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof TargetTableInfo) { + } else if (tableInfo instanceof AbstractTargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); + TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof SideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); + } else if (tableInfo instanceof AbstractSideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 78b579305..4c0d68eb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.format.dtnest; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -57,9 +57,9 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -75,7 +75,7 @@ public Row deserialize(byte[] message) throws IOException { try { for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0e8bda1fe..e49adfd93 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -92,7 +92,7 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.ObjectToMap(properties); + Map mapConf = PluginUtil.objectToMap(properties); List args = Lists.newArrayList(); for(Map.Entry one : mapConf.entrySet()){ String key = one.getKey(); diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java rename to core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java index 1fc40c13b..fbcc86bbd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java @@ -27,7 +27,7 @@ * extend RichOutputFormat with metric 'dtNumRecordsOut', 'dtNumDirtyRecordsOut', 'dtNumRecordsOutRate' * Created by sishu.yss on 2018/11/28. */ -public abstract class DtRichOutputFormat extends RichOutputFormat{ +public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index a76c1b31a..2afc76c48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TableInfoParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; @@ -75,7 +75,7 @@ public static SqlTree parseSql(String sql) throws Exception { List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); SqlTree sqlTree = new SqlTree(); - TableInfoParser tableInfoParser = new TableInfoParser(); + AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ if(Strings.isNullOrEmpty(childSql)){ continue; @@ -113,7 +113,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -126,7 +126,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -144,7 +144,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } } else { - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 1b64b7c68..5252ee022 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Maps; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class SqlTree { private Map preDealTableMap = Maps.newHashMap(); - private Map tableInfoMap = Maps.newLinkedHashMap(); + private Map tableInfoMap = Maps.newLinkedHashMap(); private List execSqlList = Lists.newArrayList(); @@ -89,11 +89,11 @@ public List getTmpSqlList(){ return tmpSqlList; } - public Map getTableInfoMap() { + public Map getTableInfoMap() { return tableInfoMap; } - public void addTableInfo(String tableName, TableInfo tableInfo){ + public void addTableInfo(String tableName, AbstractTableInfo tableInfo){ tableInfoMap.put(tableName, tableInfo); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java similarity index 96% rename from core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 8562d9859..6de2354a3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ * @author xuchao */ -public abstract class SideTableInfo extends TableInfo implements Serializable { +public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { public static final String TARGET_SUFFIX = "Side"; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java similarity index 88% rename from core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 7d16ee726..8a6851add 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -37,13 +37,13 @@ * @author xuchao */ -public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { - protected SideInfo sideInfo; + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; - public AllReqRow(SideInfo sideInfo){ + public BaseAllReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { System.out.println("----- all cacheRef init end-----"); //start reload cache thread - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java similarity index 91% rename from core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 0415102ab..ff08603cb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; @@ -47,14 +47,14 @@ * @author xuchao */ -public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); +public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; - protected SideInfo sideInfo; + protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; - public AsyncReqRow(SideInfo sideInfo){ + public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -66,12 +66,12 @@ public void open(Configuration parameters) throws Exception { } private void initCache(){ - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ return; } - AbsSideCache sideCache; + AbstractSideCache sideCache; if(ECacheType.LRU.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ sideCache = new LRUSideCache(sideTableInfo); sideInfo.setSideCache(sideCache); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java similarity index 91% rename from core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index df41e1663..9c6e1c575 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class SideInfo implements Serializable{ +public abstract class BaseSideInfo implements Serializable{ protected RowTypeInfo rowTypeInfo; @@ -66,12 +66,12 @@ public abstract class SideInfo implements Serializable{ //key:Returns the value of the position, value: the ref field name​in the side table protected Map sideFieldNameIndex = Maps.newHashMap(); - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - protected AbsSideCache sideCache; + protected AbstractSideCache sideCache; - public SideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo){ + public BaseSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo){ this.rowTypeInfo = rowTypeInfo; this.outFieldInfoList = outFieldInfoList; this.joinType = joinInfo.getJoinType(); @@ -158,7 +158,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } - public abstract void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo); + public abstract void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo); public RowTypeInfo getRowTypeInfo() { return rowTypeInfo; @@ -232,19 +232,19 @@ public void setSideFieldIndex(Map sideFieldIndex) { this.sideFieldIndex = sideFieldIndex; } - public SideTableInfo getSideTableInfo() { + public AbstractSideTableInfo getSideTableInfo() { return sideTableInfo; } - public void setSideTableInfo(SideTableInfo sideTableInfo) { + public void setSideTableInfo(AbstractSideTableInfo sideTableInfo) { this.sideTableInfo = sideTableInfo; } - public AbsSideCache getSideCache() { + public AbstractSideCache getSideCache() { return sideCache; } - public void setSideCache(AbsSideCache sideCache) { + public void setSideCache(AbstractSideCache sideCache) { this.sideCache = sideCache; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 64f9e646b..02b9c97f8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.apache.calcite.sql.SqlKind.*; @@ -46,7 +45,7 @@ * @author maqi */ public class SidePredicatesParser { - public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { + public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { SqlParser sqlParser = SqlParser.create(exeSql, CalciteConfig.MYSQL_LEX_CONFIG); SqlNode sqlNode = sqlParser.parseStmt(); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); @@ -58,7 +57,7 @@ public void fillPredicatesForSideTable(String exeSql, Map * @param sideTableMap * @param tabMapping 谓词属性中别名对应的真实维表名称 */ - private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { + private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind) { case INSERT: @@ -106,7 +105,7 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, } } - private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { + private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { predicateInfoList.stream().filter(info -> sideTableMap.containsKey(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable()))) .map(info -> sideTableMap.get(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable())).getPredicateInfoes().add(info)) .count(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 1689949ca..885fa422c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -96,7 +96,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); private StreamTableEnvironment tableEnv ; - public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, + public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); @@ -657,7 +657,7 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -665,7 +665,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { + private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -673,7 +673,7 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -704,7 +704,7 @@ public List getConditionFields(SqlNode conditionNode, String specifyTabl } public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, - Map sideTableMap, StreamTableEnvironment tableEnv, + Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache) throws Exception { @@ -778,7 +778,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, SqlNode pollSq } private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, StreamTableEnvironment tableEnv, + Map sideTableMap, StreamTableEnvironment tableEnv, List replaceInfoList) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -799,7 +799,7 @@ private void joinFun(Object pollObj, Map localTableCache, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 8417c4519..583e4597b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.PluginUtil; /** @@ -37,7 +37,7 @@ public class StreamSideFactory { private static final String CURR_TYPE = "side"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(cacheType) ? "all" : "async"; String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); @@ -45,10 +45,10 @@ public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sideParser = cl.loadClass(className); - if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { + if (!AbstractSideTableParser.class.isAssignableFrom(sideParser)) { throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); } - return sideParser.asSubclass(AbsTableParser.class).newInstance(); + return sideParser.asSubclass(AbstractTableParser.class).newInstance(); }); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java similarity index 84% rename from core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java rename to core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java index 757f91600..58832c583 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; /** * Reason: @@ -30,11 +30,11 @@ * @author xuchao */ -public abstract class AbsSideCache { +public abstract class AbstractSideCache { - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - public AbsSideCache(SideTableInfo sideTableInfo){ + public AbstractSideCache(AbstractSideTableInfo sideTableInfo){ this.sideTableInfo = sideTableInfo; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 700e13bb2..2664efd34 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,11 +33,11 @@ * @author xuchao */ -public class LRUSideCache extends AbsSideCache{ +public class LRUSideCache extends AbstractSideCache { protected transient Cache cache; - public LRUSideCache(SideTableInfo sideTableInfo) { + public LRUSideCache(AbstractSideTableInfo sideTableInfo) { super(sideTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 290804200..3c2010907 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; @@ -49,20 +49,20 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(AsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + cl.loadClass(className).asSubclass(BaseAsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 5aa810b0f..6b6f9fe1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; @@ -44,22 +44,22 @@ public class SideWithAllCacheOperator { private static final String OPERATOR_TYPE = "All"; - private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo) throws Exception { + private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(BaseAllReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); return inputStream.flatMap(allReqRow); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java index 3cfd48f1b..e29421369 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -30,5 +30,5 @@ */ public interface IStreamSinkGener { - T genStreamSink(TargetTableInfo targetTableInfo); + T genStreamSink(AbstractTargetTableInfo targetTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 53460081d..89061db3a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.table.sinks.TableSink; @@ -40,21 +40,21 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class targetParser = cl.loadClass(className); - if(!AbsTableParser.class.isAssignableFrom(targetParser)){ + if(!AbstractTableParser.class.isAssignableFrom(targetParser)){ throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); } - return targetParser.asSubclass(AbsTableParser.class).newInstance(); + return targetParser.asSubclass(AbstractTableParser.class).newInstance(); }); } - public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + public static TableSink getTableSink(AbstractTargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); diff --git a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java index b8b8f6edc..fde5a235d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -38,6 +38,6 @@ public interface IStreamSourceGener { * @param tableEnv * @return */ - T genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); + T genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index 1057fb0ed..e0cec1415 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -43,17 +43,17 @@ public class StreamSourceFactory { private static final String DIR_NAME_FORMAT = "%ssource"; - public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sourceParser = cl.loadClass(className); - if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ + if(!AbstractSourceParser.class.isAssignableFrom(sourceParser)){ throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); } - return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); + return sourceParser.asSubclass(AbstractSourceParser.class).newInstance(); }); } @@ -62,7 +62,7 @@ public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) * @param sourceTableInfo * @return */ - public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, + public static Table getStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String sqlRootDir) throws Exception { String sourceTypeStr = sourceTableInfo.getType(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java similarity index 71% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 49c5629bc..8da46b079 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; @@ -34,24 +34,24 @@ * @author xuchao */ -public abstract class AbsSideTableParser extends AbsTableParser { +public abstract class AbstractSideTableParser extends AbstractTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; private final static Pattern SIDE_TABLE_SIGN = Pattern.compile("(?i)^PERIOD\\s+FOR\\s+SYSTEM_TIME$"); - public AbsSideTableParser() { + public AbstractSideTableParser() { addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } - private void dealSideSign(Matcher matcher, TableInfo tableInfo){ + private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo){ //FIXME SIDE_TABLE_SIGN current just used as a sign for side table; and do nothing } //Analytical create table attributes ==> Get information cache - protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ - if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); + protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map props){ + if(props.containsKey(AbstractSideTableInfo.CACHE_KEY.toLowerCase())){ + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY.toLowerCase())); if(cacheType == null){ return; } @@ -61,31 +61,31 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p } sideTableInfo.setCacheType(cacheType); - if(props.containsKey(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ - Integer cacheSize = MathUtil.getIntegerVal(props.get(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ + Integer cacheSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())); if(cacheSize < 0){ throw new RuntimeException("cache size need > 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ - Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + Long cacheTTLMS = MathUtil.getLongVal(props.get(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); if(cacheTTLMS < 1000){ throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ - Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); if(partitionedJoinKey){ sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + String cachemode = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())); if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ throw new RuntimeException("cachemode must ordered or unordered!"); @@ -93,16 +93,16 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + Integer asyncCap = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())); if(asyncCap < 0){ throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + Integer asyncTimeout = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); if (asyncTimeout<0){ throw new RuntimeException("asyncTimeout size need > 0."); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java similarity index 81% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java index 745357162..308f5859f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java @@ -33,7 +33,7 @@ * @author xuchao */ -public abstract class AbsSourceParser extends AbsTableParser { +public abstract class AbstractSourceParser extends AbstractTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; private static final String WATERMARK_KEY = "waterMarkKey"; @@ -43,21 +43,21 @@ public abstract class AbsSourceParser extends AbsTableParser { private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); - public AbsSourceParser() { + public AbstractSourceParser() { addParserHandler(VIRTUAL_KEY, virtualFieldKeyPattern, this::dealVirtualField); addParserHandler(WATERMARK_KEY, waterMarkKeyPattern, this::dealWaterMark); addParserHandler(NOTNULL_KEY, notNullKeyPattern, this::dealNotNull); } - protected void dealVirtualField(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealVirtualField(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String fieldName = matcher.group(2); String expression = matcher.group(1); sourceTableInfo.addVirtualField(fieldName, expression); } - protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String eventTimeField = matcher.group(1); //FIXME Temporarily resolve the second parameter row_time_field Integer offset = MathUtil.getIntegerVal(matcher.group(3)); @@ -65,11 +65,11 @@ protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ sourceTableInfo.setMaxOutOrderness(offset); } - protected void dealNotNull(Matcher matcher, TableInfo tableInfo) { + protected void dealNotNull(Matcher matcher, AbstractTableInfo tableInfo) { String fieldName = matcher.group(1); String fieldType = matcher.group(2); Class fieldClass= dbTypeConvertToJavaType(fieldType); - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(true); tableInfo.addPhysicalMappings(fieldName, fieldName); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java index 9a41fa0a1..0fdc0e911 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java @@ -36,7 +36,7 @@ * @author xuchao */ -public abstract class SourceTableInfo extends TableInfo { +public abstract class AbstractSourceTableInfo extends AbstractTableInfo { public static final String SOURCE_SUFFIX = "Source"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java similarity index 98% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 59e4fdd39..e450adb2f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -34,7 +34,7 @@ * @author xuchao */ -public abstract class TableInfo implements Serializable { +public abstract class AbstractTableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java similarity index 85% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java index ae98d90ae..c6687433c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.StreamSideFactory; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; @@ -41,7 +41,7 @@ * @author xuchao */ -public class TableInfoParser { +public class AbstractTableInfoParser { private final static String TYPE_KEY = "type"; @@ -49,16 +49,16 @@ public class TableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); //Parsing loaded plugin - public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot) throws Exception { - AbsTableParser absTableParser = null; + public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, + String localPluginRoot) throws Exception { + AbstractTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); @@ -78,7 +78,7 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe }else{ absTableParser = sideTableInfoMap.get(type); if(absTableParser == null){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); sideTableInfoMap.put(type, absTableParser); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java similarity index 90% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 31e70caa2..52d9d9af5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -39,7 +39,7 @@ * @author xuchao */ -public abstract class AbsTableParser { +public abstract class AbstractTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; @@ -52,7 +52,7 @@ public abstract class AbsTableParser { private Map handlerMap = Maps.newHashMap(); - public AbsTableParser() { + public AbstractTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } @@ -61,9 +61,9 @@ protected boolean fieldNameNeedsUpperCase() { return true; } - public abstract TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; + public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ + public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ for(Map.Entry keyPattern : patternMap.entrySet()){ Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); @@ -82,7 +82,7 @@ public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ @@ -119,7 +119,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); @@ -131,7 +131,7 @@ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ * @param matcher * @param tableInfo */ - protected void dealNestField(Matcher matcher, TableInfo tableInfo) { + protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { String physicalField = matcher.group(1); Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); @@ -140,7 +140,7 @@ protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String mappingField = matcher.group(4); Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java index 0dc9ca77b..dbd99f993 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java @@ -29,7 +29,7 @@ * @author xuchao */ -public abstract class TargetTableInfo extends TableInfo { +public abstract class AbstractTargetTableInfo extends AbstractTableInfo { public static final String TARGET_SUFFIX = "Sink"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java index db804ea34..fab634cdc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java @@ -30,5 +30,5 @@ */ public interface ITableFieldDealHandler { - void dealPrimaryKey(Matcher matcher, TableInfo tableInfo); + void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index da5a42b57..c942fb064 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -39,7 +39,6 @@ */ public class DateUtil { - static final String timeZone = "GMT+8"; static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; static final String dateFormat = "yyyy-MM-dd"; static final String timeFormat = "HH:mm:ss"; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index a63e5a663..8eb743cf0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -109,7 +109,7 @@ public static String getSqlSideClassName(String pluginTypeName, String type, Str return CLASS_PRE_STR + "." + type.toLowerCase() + "." + pluginTypeName + "." + pluginClassName; } - public static Map ObjectToMap(Object obj) throws Exception{ + public static Map objectToMap(Object obj) throws Exception{ return objectMapper.readValue(objectMapper.writeValueAsBytes(obj), Map.class); } @@ -173,8 +173,8 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - classLoader.addURL(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + classLoader.addURL(pluginJarUrl); } } @@ -191,8 +191,8 @@ public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLExcept } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - urlList.add(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + urlList.add(pluginJarUrl); } return urlList.toArray(new URL[urlList.size()]); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java rename to core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java index ed8f64288..d75d26a61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java @@ -38,7 +38,7 @@ * @author xuchao */ -public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { +public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { private static final long serialVersionUID = 1L; @@ -55,7 +55,7 @@ public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTim protected TimeZone timezone; - public AbsCustomerWaterMarker(Time maxOutOfOrderness) { + public AbstractCustomerWaterMarker(Time maxOutOfOrderness) { super(maxOutOfOrderness); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java index 99415ba61..979b4c3d6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java @@ -35,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForLong extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForLong extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForLong.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java index b7961e410..05ee3e46e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -36,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForTimeStamp extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForTimeStamp extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForTimeStamp.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index f3208db03..b0ca0335f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; @@ -40,7 +40,7 @@ public class WaterMarkerAssigner { - public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ + public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ return false; } @@ -48,7 +48,7 @@ public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ return true; } - public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, SourceTableInfo sourceTableInfo){ + public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ String eventTimeFieldName = sourceTableInfo.getEventTimeField(); @@ -75,7 +75,7 @@ public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo type TypeInformation fieldType = fieldTypes[pos]; - AbsCustomerWaterMarker waterMarker = null; + AbstractCustomerWaterMarker waterMarker = null; if(fieldType.getTypeClass().isAssignableFrom(Timestamp.class)){ waterMarker = new CustomerWaterMarkerForTimeStamp(Time.milliseconds(maxOutOrderness), pos,timeZone); }else if(fieldType.getTypeClass().isAssignableFrom(Long.class)){ diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java index 546fcbbac..feec8e547 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java @@ -54,7 +54,7 @@ public void testfillPredicatesForSideTable() throws SqlParseException { " MyTable.a='1' and s.d='1' and s.d <> '3' and s.c LIKE '%xx%' and s.c in ('1','2') and s.c between '10' and '23' and s.d is not null\n"; - SideTableInfo sideTableInfo = new SideTableInfo(){ + AbstractSideTableInfo sideTableInfo = new AbstractSideTableInfo(){ @Override public boolean check() { return false; @@ -63,7 +63,7 @@ public boolean check() { sideTableInfo.setName("sideTable"); - Map sideTableMap = new HashMap<>(); + Map sideTableMap = new HashMap<>(); sideTableMap.put("sideTable", sideTableInfo); SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index e64e55987..b354a6269 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; @@ -43,15 +43,15 @@ public class Db2AllReqRow extends AbstractRdbAllReqRow { private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(DB2_DRIVER); - Connection conn = DriverManager.getConnection(dbURL, userName, password); + Connection conn = DriverManager.getConnection(dbUrl, userName, password); return conn; } catch (Exception e) { LOG.error("", e); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index 282ee440c..0120c58a0 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AllSideInfo extends RdbAllSideInfo { - public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 59571aa56..240bab9d1 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class Db2AsyncReqRow extends RdbAsyncReqRow { private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index e53dd7f27..9cbdc997e 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { - public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index db8a07dc6..96be4ec15 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.db2.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class Db2SideParser extends RdbSideParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index ba11aabf3..3e823ab20 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.sink.db.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -10,8 +10,8 @@ public class DbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 7ae2fb1ae..8605613d8 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -162,7 +162,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 93682de46..7988e597c 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,7 +32,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES_ADDRESS = "address"; @@ -56,7 +56,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index ac1d712ce..b6b6e0102 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -29,7 +29,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch"; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 11380eea6..6f7720472 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -45,7 +45,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; -public class HbaseAllReqRow extends AllReqRow { +public class HbaseAllReqRow extends BaseAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(HbaseAllReqRow.class); @@ -55,7 +55,7 @@ public class HbaseAllReqRow extends AllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); - public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -134,7 +134,7 @@ public void flatMap(CRow input, Collector out) throws Exception { Map cacheList = null; - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; if (hbaseSideTableInfo.isPreRowKey()) { for (Map.Entry> entry : cacheRef.get().entrySet()) { @@ -153,7 +153,7 @@ public void flatMap(CRow input, Collector out) throws Exception { } private void loadData(Map> tmpCache) throws SQLException { - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index ea51f46e4..663c2927b 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ import java.util.List; -public class HbaseAllSideInfo extends SideInfo { +public class HbaseAllSideInfo extends BaseSideInfo { private RowKeyBuilder rowKeyBuilder; - public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 251b88034..24ee01786 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -21,12 +21,12 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbsRowKeyModeDealer; +import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; @@ -58,7 +58,7 @@ * @author xuchao */ -public class HbaseAsyncReqRow extends AsyncReqRow { +public class HbaseAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; @@ -73,13 +73,13 @@ public class HbaseAsyncReqRow extends AsyncReqRow { private transient HBaseClient hBaseClient; - private transient AbsRowKeyModeDealer rowKeyMode; + private transient AbstractRowKeyModeDealer rowKeyMode; private String tableName; private String[] colNames; - public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -89,7 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List colRefType; - public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java similarity index 87% rename from hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java rename to hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 1506bc440..90ee289bd 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase.rowkeydealer; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class AbsRowKeyModeDealer { +public abstract class AbstractRowKeyModeDealer { protected Map colRefType; @@ -60,9 +60,9 @@ public abstract class AbsRowKeyModeDealer { protected Map sideFieldIndex = Maps.newHashMap(); - public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, - boolean openCache, JoinType joinType, List outFieldInfoList, - Map inFieldIndex, Map sideFieldIndex){ + public AbstractRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, + boolean openCache, JoinType joinType, List outFieldInfoList, + Map inFieldIndex, Map sideFieldIndex){ this.colRefType = colRefType; this.colNames = colNames; this.hBaseClient = hBaseClient; @@ -111,5 +111,5 @@ protected Row fillData(Row input, Object sideInput){ } public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache); + AbstractSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index a41ffe916..3fad216b2 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,7 +54,7 @@ * @author xuchao */ -public class PreRowKeyModeDealerDealer extends AbsRowKeyModeDealer { +public class PreRowKeyModeDealerDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(PreRowKeyModeDealerDealer.class); @@ -67,7 +66,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache) { + AbstractSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); prefixScanner.setFilter(scanFilter); @@ -80,7 +79,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF } - private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index b20c316db..01f43b246 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -49,7 +49,7 @@ * @author xuchao */ -public class RowKeyEqualModeDealer extends AbsRowKeyModeDealer { +public class RowKeyEqualModeDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(RowKeyEqualModeDealer.class); @@ -62,7 +62,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache){ + AbstractSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); hBaseClient.get(getRequest).addCallbacks(arg -> { diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java similarity index 93% rename from hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java rename to hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java index c3c9b5cfe..c75fa43c1 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java @@ -31,11 +31,11 @@ * @author xuchao */ -public abstract class ReplaceOperator { +public abstract class AbstractReplaceOperator { private EReplaceOpType opType; - public ReplaceOperator(EReplaceOpType opType){ + public AbstractReplaceOperator(EReplaceOpType opType){ this.opType = opType; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java index af5e24dd6..ab826c98f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java @@ -30,7 +30,7 @@ * @author xuchao */ -public class Md5ReplaceOperator extends ReplaceOperator{ +public class Md5ReplaceOperator extends AbstractReplaceOperator { public Md5ReplaceOperator(EReplaceOpType opType) { super(opType); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java index 834edf9ae..ac9a3b87c 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java @@ -28,5 +28,12 @@ */ public enum EReplaceOpType { - NO_FUNC, MD5_FUNC; + /** + * 没有func + */ + NO_FUNC, + /** + * md5 func + */ + MD5_FUNC; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index 3768d7c5e..1ebb06216 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -28,6 +28,12 @@ * @author xuchao */ public enum EReplaceType { + /** + * 参数 + */ PARAM, + /** + * 常量 + */ CONSTANT; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..cf44c181f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -20,15 +20,15 @@ package com.dtstack.flink.sql.side.hbase.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * hbase field information must include the definition of an alias -> sql which does not allow ":" @@ -37,7 +37,7 @@ * @author xuchao */ -public class HbaseSideParser extends AbsSideTableParser { +public class HbaseSideParser extends AbstractSideTableParser { private final static String FIELD_KEY = "fieldKey"; @@ -58,7 +58,7 @@ public HbaseSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); @@ -77,7 +77,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 447b02921..9308725a2 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,7 +30,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -59,7 +58,7 @@ public HbaseSink() { } @Override - public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { + public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { HbaseTableInfo hbaseTableInfo = (HbaseTableInfo) targetTableInfo; this.zookeeperQuorum = hbaseTableInfo.getHost(); this.port = hbaseTableInfo.getPort(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index f1ae72b6b..d09f54128 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; @@ -31,14 +31,14 @@ import java.util.List; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Date: 2018/09/14 * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseSinkParser extends AbsTableParser { +public class HbaseSinkParser extends AbstractTableParser { public static final String HBASE_ZOOKEEPER_QUORUM = "zookeeperQuorum"; @@ -59,7 +59,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseTableInfo hbaseTableInfo = new HbaseTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 610eb5c30..62f7c9936 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import java.util.Map; @@ -30,7 +30,7 @@ * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseTableInfo extends TargetTableInfo { +public class HbaseTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "hbase"; diff --git a/hbase/pom.xml b/hbase/pom.xml index 835457e29..f64ffab1b 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,6 +29,13 @@ 1.0-SNAPSHOT provided + + jdk.tools + jdk.tools + 1.6 + system + ${JAVA_HOME}/lib/tools.jar + org.apache.hbase diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 4118421ee..10938308a 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; @@ -53,13 +53,13 @@ public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private ImpalaSideTableInfo impalaSideTableInfo; - public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.impalaSideTableInfo = (ImpalaSideTableInfo) sideTableInfo; } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Connection connection ; String url = getUrl(); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java index d40c5f48c..a5e643967 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java @@ -20,10 +20,9 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -32,7 +31,7 @@ public class ImpalaAllSideInfo extends RdbAllSideInfo { - public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -61,7 +60,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 1b983ca2c..77d3f3007 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import io.vertx.core.Vertx; @@ -50,7 +50,7 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java index 53559b9a7..14fbf0ed3 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -38,7 +38,7 @@ public class ImpalaAsyncSideInfo extends RdbAsyncSideInfo { - public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -69,7 +69,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 344037124..10a13ec72 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,14 +19,13 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Arrays; import java.util.HashMap; @@ -48,7 +47,7 @@ public class ImpalaSideParser extends RdbSideParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaSideTableInfo impalaSideTableInfo = new ImpalaSideTableInfo(); impalaSideTableInfo.setType(CURR_TYPE); impalaSideTableInfo.setName(tableName); @@ -147,6 +146,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 91b6532bf..cd1e1e945 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 4921f5e51..5b6fdeafe 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -19,12 +19,11 @@ package com.dtstack.flink.sql.sink.impala.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -42,7 +41,7 @@ public class ImpalaSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); @@ -117,6 +116,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4fb567f82..c6cce594c 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.sink.kafka.table; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,9 +32,9 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkParser extends AbsTableParser { +public class KafkaSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); kafkaSinkTableInfo.setName(tableName); kafkaSinkTableInfo.setType(MathUtil.getString(props.get(KafkaSinkTableInfo.TYPE_KEY.toLowerCase()))); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java index 6ac728ee0..ec554d5f1 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java @@ -18,8 +18,7 @@ package com.dtstack.flink.sql.sink.kafka.table; -import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -33,7 +32,7 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkTableInfo extends TargetTableInfo { +public class KafkaSinkTableInfo extends AbstractTargetTableInfo { public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 10049cbbe..8b0b9753c 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,9 +32,9 @@ * * @author sishu.yss */ -public class KafkaSourceParser extends AbsSourceParser { +public class KafkaSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); kafkaSourceTableInfo.setName(tableName); @@ -43,7 +43,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map, IStreamSinkGener< protected Optional> partitioner; @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafkaSinkTableInfo.getTopic(); diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index a445a6529..7e2d77c27 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -56,7 +56,7 @@ public class KafkaSource implements IStreamSourceGener { */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -91,7 +91,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 6d39536a5..d8c66cbc8 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -68,7 +68,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka09SinkTableInfo.getTopic(); diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 21ed7c3e3..ffb466a4c 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -59,7 +59,7 @@ public class KafkaSource implements IStreamSourceGener
{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -94,7 +94,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for(Map.Entry entry:offsetMap.entrySet()){ specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index dee05ab08..67df2159c 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -69,7 +69,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Optional> partitioner; @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka10SinkTableInfo.getTopic(); diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 55cffbf1e..ff4aed89d 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -60,7 +60,7 @@ public class KafkaSource implements IStreamSourceGener
{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -94,7 +94,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index c2ef7d35a..a3533b7aa 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -70,7 +70,7 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka11SinkTableInfo.getTopic(); diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 1a4eed08a..11be1898a 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -60,7 +60,7 @@ public class KafkaSource implements IStreamSourceGener
{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); @@ -96,7 +96,7 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); + Map offsetMap = PluginUtil.objectToMap(properties); Map specificStartupOffsets = new HashMap<>(); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index cf2c70f1f..45b312ffa 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.side.kudu; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; import org.apache.calcite.sql.JoinType; @@ -39,7 +39,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -public class KuduAllReqRow extends AllReqRow { +public class KuduAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 6051774809356082219L; @@ -60,7 +60,7 @@ public class KuduAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java index ddfa73201..1241a6f37 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -12,16 +12,16 @@ import java.util.List; -public class KuduAllSideInfo extends SideInfo { +public class KuduAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 9005389633060174746L; - public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { // no use } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 12689a07b..1cdc75e1b 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; -public class KuduAsyncReqRow extends AsyncReqRow { +public class KuduAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); /** @@ -52,7 +52,7 @@ public class KuduAsyncReqRow extends AsyncReqRow { private AsyncKuduScanner.AsyncKuduScannerBuilder scannerBuilder; - public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index de75f23fb..5ace515f7 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -12,15 +12,15 @@ import java.util.List; -public class KuduAsyncSideInfo extends SideInfo { +public class KuduAsyncSideInfo extends BaseSideInfo { - public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index 83864fa22..62d215d87 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -9,9 +9,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSideParser extends AbsSideTableParser { +public class KuduSideParser extends AbstractSideTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -52,7 +52,7 @@ public class KuduSideParser extends AbsSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduSideTableInfo kuduSideTableInfo = new KuduSideTableInfo(); kuduSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduSideTableInfo); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java index c527ec268..eb8df5ad9 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java @@ -1,9 +1,9 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; -public class KuduSideTableInfo extends SideTableInfo { +public class KuduSideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 780319d82..c211c7a24 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -42,7 +42,7 @@ * @author gituser * @modify xiuzhu */ -public class KuduOutputFormat extends DtRichOutputFormat { +public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 0c90ffe94..4c7c3bea8 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -3,7 +3,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kudu.table.KuduTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +39,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; @Override - public KuduSink genStreamSink(TargetTableInfo targetTableInfo) { + public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; this.kuduMasters = kuduTableInfo.getKuduMasters(); this.tableName = kuduTableInfo.getTableName(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index e38a85390..20302d44f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,8 +1,8 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -10,9 +10,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSinkParser extends AbsTableParser { +public class KuduSinkParser extends AbstractTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -27,7 +27,7 @@ public class KuduSinkParser extends AbsTableParser { public static final String SOCKET_READ_TIMEOUT_MS = "defaultSocketReadTimeoutMs"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduTableInfo kuduTableInfo = new KuduTableInfo(); kuduTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduTableInfo); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java index c086c7298..80e47761b 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; -public class KuduTableInfo extends TargetTableInfo { +public class KuduTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 3229101c6..79f5bd9ca 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,10 +18,10 @@ package com.dtstack.flink.sql.side.mongo; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; @@ -57,7 +57,7 @@ * * @author xuqianjin */ -public class MongoAllReqRow extends AllReqRow { +public class MongoAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = -675332795591842778L; @@ -73,7 +73,7 @@ public class MongoAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index b2d5c2686..4a33f1a69 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -36,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAllSideInfo extends SideInfo{ +public class MongoAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -1696860430075523841L; - public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 5ef06c80e..491f62922 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -20,27 +20,22 @@ package com.dtstack.flink.sql.side.mongo; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.mongodb.BasicDBObject; import com.mongodb.Block; import com.mongodb.ConnectionString; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; import com.mongodb.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import com.mongodb.connection.ClusterSettings; -import com.mongodb.connection.ConnectionPoolSettings; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import org.apache.flink.configuration.Configuration; @@ -53,7 +48,6 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -66,7 +60,7 @@ * * @author xuqianjin */ -public class MongoAsyncReqRow extends AsyncReqRow { +public class MongoAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -1183158242862673706L; private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); @@ -77,7 +71,7 @@ public class MongoAsyncReqRow extends AsyncReqRow { private MongoSideTableInfo mongoSideTableInfo; - public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index de81ed1f1..19534d4e1 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -20,16 +20,13 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import java.util.List; @@ -39,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAsyncSideInfo extends SideInfo { +public class MongoAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -3694857194254465989L; - public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 5edc12419..4fe1ebee1 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -19,15 +19,13 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -35,7 +33,7 @@ * * @author xuqianjin */ -public class MongoSideParser extends AbsSideTableParser { +public class MongoSideParser extends AbstractSideTableParser { public static final String ADDRESS_KEY = "address"; @@ -48,7 +46,7 @@ public class MongoSideParser extends AbsSideTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoSideTableInfo mongoSideTableInfo = new MongoSideTableInfo(); mongoSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoSideTableInfo); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index a5c834469..c83de7bbc 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class MongoSideTableInfo extends SideTableInfo { +public class MongoSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index daaded15e..03ef8f3b3 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; @@ -41,7 +41,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends DtRichOutputFormat { +public class MongoOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 4e28d8fd2..3f34055ec 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.mongo.table.MongoTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -54,7 +54,7 @@ public MongoSink() { } @Override - public MongoSink genStreamSink(TargetTableInfo targetTableInfo) { + public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { MongoTableInfo mongoTableInfo = (MongoTableInfo) targetTableInfo; this.address = mongoTableInfo.getAddress(); this.tableName = mongoTableInfo.getTableName(); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java index d4810fb6d..f093b70aa 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -34,7 +34,7 @@ * @author xuqianjin */ -public class MongoSinkParser extends AbsTableParser { +public class MongoSinkParser extends AbstractTableParser { private static final String CURR_TYPE = "mongo"; @@ -49,7 +49,7 @@ public class MongoSinkParser extends AbsTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoTableInfo mongoTableInfo = new MongoTableInfo(); mongoTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoTableInfo); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index 02a96d6bb..c9d94dfe6 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -30,7 +30,7 @@ */ -public class MongoTableInfo extends TargetTableInfo { +public class MongoTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "mongo"; diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index 8cf6ab597..eb294e64e 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -49,7 +49,7 @@ public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 81193c0f9..89a62a5ef 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class MysqlAllSideInfo extends RdbAllSideInfo { - public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 2d2f5a72a..290f02e2b 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -49,7 +49,7 @@ public class MysqlAsyncReqRow extends RdbAsyncReqRow { private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index f72671ce0..8fe10d06d 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { - public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index 40f68e7e4..b0a38e344 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class MysqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 2247eb8cc..49105a7a8 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mysql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class MysqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 5477514a5..3be6687ec 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -42,7 +42,7 @@ public class OracleAllReqRow extends AbstractRdbAllReqRow { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 361366929..342533681 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -19,18 +19,17 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; public class OracleAllSideInfo extends RdbAllSideInfo { - public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index f6827f41b..c0b37e7ac 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -42,7 +42,7 @@ public class OracleAsyncReqRow extends RdbAsyncReqRow { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index b811cf783..8c620b013 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -20,22 +20,18 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.ParseUtils; -import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import java.util.Arrays; import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index f9124bd10..897b77d30 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.oracle.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -27,8 +27,8 @@ public class OracleSideParser extends RdbSideParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index aff096bd3..1732c8cc2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.oracle.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class OracleSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index a5eec511b..5fbf0824b 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; @@ -45,7 +45,7 @@ public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index fa2ad63e7..8f30c8dd3 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,7 +31,7 @@ * @author yinxi */ public class PolardbAllSideInfo extends RdbAllSideInfo { - public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 0f8f82738..286f17286 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -44,7 +44,7 @@ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private final static String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index f6afb8da8..b52d52390 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -32,7 +32,7 @@ */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { - public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 1afb661f0..71aad7cbb 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -32,8 +32,8 @@ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index a57089941..b4f02665e 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -31,8 +31,8 @@ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; } diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 1c9aecfe1..0ec8f96ce 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -49,7 +49,7 @@ public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index d383ee46d..a4a52eabc 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index a26c8b229..2a9e0137c 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index 1d89f4894..f19a488bc 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index faee2c704..be0c5ea8b 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.postgresql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class PostgresqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index 5943b5e52..e94cfc8a0 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -35,8 +35,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index aeb88f7a3..e2452f73b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.rdb.all; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import org.apache.calcite.sql.JoinType; @@ -51,7 +51,7 @@ * @author maqi */ -public abstract class AbstractRdbAllReqRow extends AllReqRow { +public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 2098635140857937718L; @@ -61,7 +61,7 @@ public abstract class AbstractRdbAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public AbstractRdbAllReqRow(SideInfo sideInfo) { + public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 619d08529..1751b12aa 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -43,17 +43,17 @@ * @author maqi */ -public class RdbAllSideInfo extends SideInfo { +public class RdbAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -5858335638589472159L; - public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(sideSelectFields.split(",")), sideTableInfo.getPredicateInfoes()); System.out.println("--------dimension sql query-------\n" + sqlCondition); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index bf7f8f774..f78c30c30 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,9 +20,9 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import io.vertx.core.json.JsonArray; @@ -47,7 +47,7 @@ * @author maqi */ -public class RdbAsyncReqRow extends AsyncReqRow { +public class RdbAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; @@ -69,7 +69,7 @@ public class RdbAsyncReqRow extends AsyncReqRow { private transient SQLClient rdbSqlClient; - public RdbAsyncReqRow(SideInfo sideInfo) { + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 0fe3eb0b9..d79d27460 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -34,7 +34,6 @@ import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; @@ -46,16 +45,16 @@ * @author maqi */ -public class RdbAsyncSideInfo extends SideInfo { +public class RdbAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1942629132469918611L; - public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index b9811b0ee..79942ae11 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,10 +33,10 @@ * @author maqi */ -public class RdbSideParser extends AbsSideTableParser { +public class RdbSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbSideTableInfo rdbTableInfo = new RdbSideTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 0dfbef325..506033007 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class RdbSideTableInfo extends SideTableInfo { +public class RdbSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index b7990f9f4..bdc7e3dbf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -85,7 +85,7 @@ public AbstractRdbSink(JDBCDialect jdbcDialect) { } @Override - public AbstractRdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 3378ff3f2..2a3ce5e90 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; @@ -36,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { +public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index 9bbff3b4d..aca7acb6c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,9 +17,8 @@ */ package com.dtstack.flink.sql.sink.rdb.table; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -31,9 +30,9 @@ * * @author maqi */ -public class RdbSinkParser extends AbsTableParser { +public class RdbSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbTableInfo rdbTableInfo = new RdbTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index d5ad4eab6..cf185308c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.sink.rdb.table; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; - /** * Reason: * Date: 2018/11/27 @@ -31,7 +29,7 @@ * * @author maqi */ -public class RdbTableInfo extends TargetTableInfo { +public class RdbTableInfo extends AbstractTargetTableInfo { public static final String URL_KEY = "url"; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 373a59f6a..a2a915bb1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -59,7 +59,7 @@ public static AbstractUpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -97,9 +97,9 @@ public static AbstractUpsertWriter create( private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { + private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, AbstractDtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; @@ -217,7 +217,7 @@ private UpsertWriterUsingUpsertStatement( boolean objectReuse, String deleteSql, String upsertSql, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); this.upsertSql = upsertSql; } @@ -275,7 +275,7 @@ private UpsertWriterUsingInsertUpdateStatement( String existSql, String insertSql, String updateSql, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); this.existSql = existSql; this.insertSql = insertSql; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 2aa7b01f9..341ecf983 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -48,9 +48,9 @@ public class AppendOnlyWriter implements JDBCWriter { private transient PreparedStatement statement; private transient List rows; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSql, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { + public AppendOnlyWriter(String insertSql, int[] fieldTypes, AbstractDtRichOutputFormat metricOutputFormat) { this.insertSql = insertSql; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index a215c9d75..8f884ca03 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -42,7 +42,7 @@ /** * @author yanxi */ -public class RedisAllReqRow extends AllReqRow{ +public class RedisAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 7578879189085344807L; @@ -60,7 +60,7 @@ public class RedisAllReqRow extends AllReqRow{ private RedisSideReqRow redisSideReqRow; - public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index fc24bc4ef..ec0fa0d15 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ /** * @author yanxi */ -public class RedisAllSideInfo extends SideInfo { +public class RedisAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1998703966487857613L; - public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index ee951a928..119ff5634 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -46,7 +46,7 @@ /** * @author yanxi */ -public class RedisAsyncReqRow extends AsyncReqRow { +public class RedisAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -64,7 +64,7 @@ public class RedisAsyncReqRow extends AsyncReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index 2736fc8a7..c23035ec7 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,15 +31,15 @@ /** * @author yanxi */ -public class RedisAsyncSideInfo extends SideInfo { +public class RedisAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4851348392924455039L; - public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index 2d0fb77d4..514984f0a 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -27,10 +27,10 @@ /** * @author yanxi */ -public class RedisSideParser extends AbsSideTableParser { +public class RedisSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisSideTableInfo redisSideTableInfo = new RedisSideTableInfo(); redisSideTableInfo.setName(tableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 5007868e9..17557f916 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.ISideReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,9 +40,9 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; - private SideInfo sideInfo; + private BaseSideInfo sideInfo; - public RedisSideReqRow(SideInfo sideInfo){ + public RedisSideReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -72,7 +72,7 @@ public Row fillData(Row input, Object sideInput) { return row; } - public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index ae6cdd017..5224134c8 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -18,12 +18,12 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** * @author yanxi */ -public class RedisSideTableInfo extends SideTableInfo { +public class RedisSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index bbf49dff7..a51c62417 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.redis; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -43,7 +43,7 @@ /** * @author yanxi */ -public class RedisOutputFormat extends DtRichOutputFormat { +public class RedisOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index 007c7fd3b..cc49a3ba8 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.redis.table.RedisTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -69,7 +69,7 @@ public RedisSink(){ } @Override - public RedisSink genStreamSink(TargetTableInfo targetTableInfo) { + public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RedisTableInfo redisTableInfo = (RedisTableInfo) targetTableInfo; this.url = redisTableInfo.getUrl(); this.database = redisTableInfo.getDatabase(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 201abab2c..825d6901e 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -30,9 +30,9 @@ /** * @author yanxi */ -public class RedisSinkParser extends AbsTableParser { +public class RedisSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisTableInfo redisTableInfo = new RedisTableInfo(); redisTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, redisTableInfo); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 5c9940252..2425f8c38 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -18,12 +18,12 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** * @author yanxi */ -public class RedisTableInfo extends TargetTableInfo { +public class RedisTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index 2f508b3e5..b032fdce6 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.types.Row; @@ -67,7 +67,7 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { ServersocketSourceTableInfo tableInfo; public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, - Map rowAndFieldMapping, List fieldExtraInfos) { + Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java index 3ab01fb42..b4dda5f3b 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ */ public class ServersocketSource implements IStreamSourceGener
{ @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; String tableName = serversocketSourceTableInfo.getName(); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java index b52e38e59..f2d861b17 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java @@ -17,8 +17,8 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +30,9 @@ * * @author maqi */ -public class ServersocketSourceParser extends AbsSourceParser { +public class ServersocketSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); serversocketSourceTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 02f6e9f30..3123b477f 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class ServersocketSourceTableInfo extends SourceTableInfo { +public class ServersocketSourceTableInfo extends AbstractSourceTableInfo { private static final String CURR_TYPE = "serversocket"; diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index af1d94171..dbae271e4 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -46,7 +46,7 @@ public class SqlserverAllReqRow extends AbstractRdbAllReqRow { private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index 8f0fc63a4..a40af5697 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -31,7 +31,7 @@ */ public class SqlserverAllSideInfo extends RdbAllSideInfo { - public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index 070bfe334..f78061fca 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -44,7 +44,7 @@ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index d3c833c96..e8a487721 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,7 +34,7 @@ */ public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { - public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index be8ebb152..e06c13898 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.sqlserver.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; /** @@ -30,8 +30,8 @@ public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index a695d6c3b..5300884bc 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.sqlserver.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class SqlserverSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; } From 9ae4741974591282538afc6c52dc480f2eb5f09f Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 5 Mar 2020 14:54:33 +0800 Subject: [PATCH 019/378] es id null check --- .../sql/sink/elasticsearch/CustomerSinkFunc.java | 13 ++++++++----- .../sql/sink/elasticsearch/CustomerSinkFunc.java | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 1f3efb8d7..0d97f5995 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -96,11 +96,14 @@ public void setOutRecords(Counter outRecords) { } private IndexRequest createIndexRequest(Row element) { - // index start at 1, - String idFieldStr = idFieldIndexList.stream() - .filter(index -> index > 0 && index <= element.getArity()) - .map(index -> element.getField(index - 1).toString()) - .collect(Collectors.joining(ID_VALUE_SPLIT)); + String idFieldStr = ""; + if (null != idFieldIndexList) { + // index start at 1, + idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); + } Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index b1c8a5e97..ec1732d0e 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -95,11 +95,14 @@ public void setOutDirtyRecords(Counter outDirtyRecords) { } private IndexRequest createIndexRequest(Row element) { - // index start at 1, - String idFieldStr = idFieldIndexList.stream() - .filter(index -> index > 0 && index <= element.getArity()) - .map(index -> element.getField(index - 1).toString()) - .collect(Collectors.joining(ID_VALUE_SPLIT)); + String idFieldStr = ""; + if (null != idFieldIndexList) { + // index start at 1, + idFieldStr = idFieldIndexList.stream() + .filter(index -> index > 0 && index <= element.getArity()) + .map(index -> element.getField(index - 1).toString()) + .collect(Collectors.joining(ID_VALUE_SPLIT)); + } Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); From 228c935e6a62f7f875805f0dad3f3881698c4021 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 5 Mar 2020 17:13:43 +0800 Subject: [PATCH 020/378] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=9F=BA=E6=9C=AC?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 7 ++++++- core/src/main/java/com/dtstack/flink/sql/Main.java | 1 + .../dtstack/flink/sql/constrant/ConfigConstrant.java | 3 ++- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 11 ++++++++++- launcher/pom.xml | 5 ----- .../com/dtstack/flink/sql/launcher/LauncherMain.java | 8 -------- pom.xml | 1 - 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 677f9d5ee..eb470a3c0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,6 +20,7 @@ 1.16.0 2.7.9 19.0 + 1.0.0-SNAPSHOT @@ -121,7 +122,11 @@ junit 4.12 - + + com.aiweiergou + tools-logger + ${logger.tool.version} + diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index 5d7528869..fe925f9a6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -38,6 +38,7 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 8e5f34d80..d1885a570 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -65,7 +65,7 @@ public class ConfigConstrant { public static final String RESTOREENABLE = "restore.enable"; - public static final String LOG_LEVEL_KEY = "logLevel"; + public static final String LOG_LEVEL_KEY = "logLevel"; // restart plocy @@ -75,4 +75,5 @@ public class ConfigConstrant { public static final String DELAYINTERVAL= "delay.interval"; //sec + } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5cd1248e2..648745409 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,8 +18,10 @@ package com.dtstack.flink.sql.exec; +import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.config.CalciteConfig; +import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -353,5 +355,12 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - + public static void setLogLevel(ParamsInfo paramsInfo){ + String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); + if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ + return; + } + ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); + logLevelProcess.process(logLevel); + } } \ No newline at end of file diff --git a/launcher/pom.xml b/launcher/pom.xml index 8c1ce53d5..1771df1ec 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -34,11 +34,6 @@ fastjson 1.2.7 - - com.aiweiergou - tools-logger - ${logger.tool.version} - diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 95afc86cd..f8338d2d4 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -149,12 +149,4 @@ private static String[] parseJson(String[] args) { String[] array = list.toArray(new String[list.size()]); return array; } - - private static void setLogLevel(String logLevel){ - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } diff --git a/pom.xml b/pom.xml index b0ba9f47d..5493645e9 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,6 @@ UTF-8 1.8.1 - 1.0.0-SNAPSHOT From 02e896e35a5cf597c85b4ff06314519a898dcf92 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 5 Mar 2020 17:21:57 +0800 Subject: [PATCH 021/378] =?UTF-8?q?=E7=A7=BB=E9=99=A4launcher=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/launcher/LauncherMain.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index f8338d2d4..0b036c87c 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -84,8 +84,6 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - setLogLevel(confProperties.getProperty(ConfigConstrant.LOG_LEVEL_KEY)); - if(mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[argList.size()]); Main.main(localArgs); From 2a0663e8302fe3111c310183f56e3c5e1d27717d Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 6 Mar 2020 13:14:47 +0800 Subject: [PATCH 022/378] codeview --- .../com/dtstack/flink/sql/classloader/ClassLoaderManager.java | 4 ++-- .../java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 4 ++-- .../main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java | 4 ++-- .../main/java/com/dtstack/flink/sql/side/ParserJoinField.java | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index c03f60617..2e62e11ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -98,9 +98,9 @@ public static List getClassPath() { - public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) + public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) throws IllegalAccessException, InvocationTargetException { - for(URL url : jarURLList){ + for(URL url : jarUrlList){ if(url.toString().endsWith(".jar")){ urlClassLoaderAddUrl(classLoader, url); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index ed54a9c48..40729d257 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -117,7 +117,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - List jarURList = getExternalJarUrls(options.getAddjar()); + List jarUrlList = getExternalJarUrls(options.getAddjar()); return ParamsInfo.builder() .setSql(sql) @@ -127,7 +127,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarURList) + .setJarUrlList(jarUrlList) .setLogLevel(logLevel) .build(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 8a6851add..357c65571 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -26,8 +26,8 @@ import org.apache.flink.table.runtime.types.CRow; import java.sql.SQLException; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -60,7 +60,7 @@ public void open(Configuration parameters) throws Exception { //start reload cache thread AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); - es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); + es = new ScheduledThreadPoolExecutor(1,new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index 738e0f84e..d9d79f34e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -80,6 +80,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b switch(identifierSize) { case 1: fieldInfoList.addAll(getAllField(scope)); + break; default: SqlIdentifier tableIdentify = identifier.skipLast(1); JoinScope.ScopeChild scopeChild = scope.getScope(tableIdentify.getSimple()); @@ -99,6 +100,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b fieldInfo.setTypeInformation(type); fieldInfoList.add(fieldInfo); } + break; } } } From 78b3a1a4eab7812892e15ed2f773b5002c2ede0f Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 6 Mar 2020 17:05:29 +0800 Subject: [PATCH 023/378] =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/util/DateUtil.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index c942fb064..462eed30b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -23,6 +23,12 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Calendar; import java.util.Date; import java.util.Locale; @@ -39,12 +45,9 @@ */ public class DateUtil { - static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; - static final String dateFormat = "yyyy-MM-dd"; - static final String timeFormat = "HH:mm:ss"; - static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); - static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { @@ -69,23 +72,32 @@ public static Date stringToDate(String strDate) { return null; } try { - return datetimeFormatter.parse(strDate); - } catch (ParseException ignored) { + ; + return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); + } catch (DateTimeParseException ignored) { } try { - return dateFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); + } catch (DateTimeParseException ignored) { } try { - return timeFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); + } catch (DateTimeParseException ignored) { } throw new RuntimeException("can't parse date"); } + public static Date localDateTimetoDate(LocalDateTime localDateTime){ + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime dateToLocalDateTime(Date date){ + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + /** * * @@ -762,11 +774,12 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - return dateFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_FORMATTER); } public static String timestampToString(Date date) { - return datetimeFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_TIME_FORMATTER); } - } From 02f2c37407225332bfcadb05f279e7042c3be5f9 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 6 Mar 2020 17:12:50 +0800 Subject: [PATCH 024/378] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E9=81=97=E6=BC=8F?= =?UTF-8?q?=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/console/table/TablePrintUtil.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index c0d35404d..6ddc60386 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -22,6 +22,7 @@ public class TablePrintUtil { public static final int ALIGN_LEFT = 1;//左对齐 public static final int ALIGN_RIGHT = 2;//右对齐 public static final int ALIGN_CENTER = 3;//居中对齐 + private static final Pattern PATTERN = Pattern.compile("[\u4e00-\u9fa5]"); private int align = ALIGN_CENTER;//默认居中对齐 private boolean equilong = false;//默认不等宽 @@ -138,8 +139,8 @@ private static ListgetColList(Object obj) { * @return */ private int getStringCharLength(String str) { - Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 - Matcher m = p.matcher(str); + //利用正则找到中文 + Matcher m = PATTERN.matcher(str); int count = 0; while (m.find()) { count++; @@ -219,6 +220,8 @@ public String getTableString() { sb.append(cell); for (int i = 0; i < right + padding; i++) {sb.append(s);} break; + default: + break; } sb.append(v); } From 4c7f9867045c5e3f94906e891ca354b6753561fb Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 10 Mar 2020 15:45:22 +0800 Subject: [PATCH 025/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 2 +- .../dtstack/flink/sql/side/SideSqlExec.java | 4 ++-- .../sql/table/AbstractSideTableParser.java | 4 ++-- .../sink/elasticsearch/ElasticsearchSink.java | 2 +- .../Elasticsearch6AllReqRow.java | 21 ++++++++----------- .../Elasticsearch6AllSideInfo.java | 11 +++++----- .../Elasticsearch6AsyncReqRow.java | 4 ++-- .../Elasticsearch6AsyncSideInfo.java | 13 ++++++------ .../table/Elasticsearch6SideParser.java | 8 +++---- .../table/Elasticsearch6SideTableInfo.java | 4 ++-- .../sql/side/elasticsearch6/util/Es6Util.java | 10 ++++----- .../sink/elasticsearch/ElasticsearchSink.java | 4 ++-- .../table/ElasticsearchSinkParser.java | 8 +++---- .../table/ElasticsearchTableInfo.java | 4 ++-- .../sql/sink/hbase/HbaseOutputFormat.java | 7 +------ .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/sink/kafka/KafkaSink.java | 1 + .../flink/sql/side/mongo/MongoAllReqRow.java | 5 +++-- .../side/rdb/all/AbstractRdbAllReqRow.java | 3 +-- .../sink/rdb/writer/AbstractUpsertWriter.java | 10 ++++----- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 2 +- .../sql/side/redis/RedisAsyncReqRow.java | 4 ++-- 24 files changed, 65 insertions(+), 69 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 85568f19e..c02c3b142 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -127,7 +127,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarURList) + .setJarUrlList(jarUrlList) .build(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index a92685f3c..e828bec03 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -99,7 +99,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); - public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, + public void exec(String sql, Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, CreateTmpTableParser.SqlParserResult createView) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); @@ -732,7 +732,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, Map tableCache, List replaceInfoList) throws SqlParseException { - AliasInfo aliasInfo = parseASNode(pollSqlNode); + AliasInfo aliasInfo = parseAsNode(pollSqlNode); if (localTableCache.containsKey(aliasInfo.getName())) { return; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index a168eebbc..39b72c22f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -108,8 +108,8 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 4229abb99..5ca81c5ed 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,7 +34,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 648f09fba..b8cf2c46a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -18,24 +18,22 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -49,7 +47,6 @@ import java.io.IOException; import java.io.Serializable; -import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; import java.util.List; @@ -60,7 +57,7 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { +public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); @@ -70,7 +67,7 @@ public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private SearchRequest searchRequest; private BoolQueryBuilder boolQueryBuilder; - public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index bf21b3ca8..d353a583d 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -18,12 +18,13 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; + import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; @@ -35,15 +36,15 @@ * @author yinxi * @date 2020/1/13 - 1:01 */ -public class Elasticsearch6AllSideInfo extends SideInfo { +public class Elasticsearch6AllSideInfo extends BaseSideInfo { - public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index f67177fe4..e8164edb2 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -57,14 +57,14 @@ * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { +public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); SqlNode conditionNode = joinInfo.getCondition(); ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index 7b3a2f9e7..4118988ab 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -18,18 +18,17 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -37,15 +36,15 @@ * @author yinxi * @date 2020/2/13 - 13:09 */ -public class Elasticsearch6AsyncSideInfo extends SideInfo { +public class Elasticsearch6AsyncSideInfo extends BaseSideInfo { - public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index 13dfe2995..1b39bbf0f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -30,7 +30,7 @@ * @author yinxi * @date 2020/1/13 - 1:07 */ -public class Elasticsearch6SideParser extends AbsSideTableParser { +public class Elasticsearch6SideParser extends AbstractSideTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); elasticsearch6SideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java index 0afe2d59e..a01c90a53 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -26,7 +26,7 @@ * @author yinxi * @date 2020/1/13 - 15:00 */ -public class Elasticsearch6SideTableInfo extends SideTableInfo { +public class Elasticsearch6SideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index cc591789e..3d9e4d62a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; @@ -98,7 +98,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } // add index and type to search request - public static SearchRequest setSearchRequest(SideInfo sideInfo) { + public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); // determine existence of index @@ -129,7 +129,7 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { } // build where cause - public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { + public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -144,7 +144,7 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { } // build filter condition - public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, BaseSideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); @@ -202,7 +202,7 @@ public static String[] removeSpaceAndApostrophe(String str) { } // prevent word segmentation - public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { + public static String textConvertToKeyword(String fieldName, BaseSideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); String fieldType = sideFieldTypes[fieldIndex]; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 2646c50e9..b7d9de6fc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,7 +32,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -128,7 +128,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; clusterName = esTableInfo.getClusterName(); index = esTableInfo.getIndex(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 8902b953c..22c2b72bc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 3cc3dd9ff..d0eef18f0 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 7709e68d9..382e8f975 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -20,19 +20,14 @@ package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index dbd231dfe..7105bc037 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 7dad2896f..0a991a8ea 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 5144f4486..34ea8fc5f 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 3c5d638cc..d7807a935 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 93b5a379f..3373f3667 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -35,6 +35,7 @@ import com.mongodb.client.MongoDatabase; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -215,7 +216,7 @@ private void loadData(Map>> tmpCache) throws SQ String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); BasicDBObject basicDBObject = new BasicDBObject(); for (String selectField : sideFieldNames) { - basicDbObject.append(selectField, 1); + basicDBObject.append(selectField, 1); } BasicDBObject filterObject = new BasicDBObject(); try { @@ -232,7 +233,7 @@ private void loadData(Map>> tmpCache) throws SQ } - FindIterable findIterable = dbCollection.find(filterObject).projection(basicDbObject).limit(FETCH_SIZE); + FindIterable findIterable = dbCollection.find(filterObject).projection(basicDBObject).limit(FETCH_SIZE); MongoCursor mongoCursor = findIterable.iterator(); while (mongoCursor.hasNext()) { Document doc = mongoCursor.next(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 339a4a3e8..17daecfb0 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -24,8 +24,7 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; + import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 481cd27d6..9ca0fd754 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -116,7 +116,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.deleteStatement = connection.prepareStatement(deleteSQL); + this.deleteStatement = connection.prepareStatement(deleteSql); } @Override @@ -235,7 +235,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - upsertStatement = connection.prepareStatement(upsertSQL); + upsertStatement = connection.prepareStatement(upsertSql); } @Override @@ -300,9 +300,9 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - existStatement = connection.prepareStatement(existSQL); - insertStatement = connection.prepareStatement(insertSQL); - updateStatement = connection.prepareStatement(updateSQL); + existStatement = connection.prepareStatement(existSql); + insertStatement = connection.prepareStatement(insertSql); + updateStatement = connection.prepareStatement(updateSql); } @Override diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 5c9fcf8ac..3559d4376 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -64,7 +64,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.statement = connection.prepareStatement(insertSQL); + this.statement = connection.prepareStatement(insertSql); } /** diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index ba6dc51ab..d2a2aed56 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,6 +18,8 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -25,11 +27,9 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; From b78a00511222379d99f2b7cb19f3ac63dbfd2779 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 10 Mar 2020 16:09:31 +0800 Subject: [PATCH 026/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/cassandra/CassandraOutputFormat.java | 2 +- .../flink/sql/side/kudu/KuduAllReqRow.java | 6 ++++++ .../flink/sql/side/kudu/KuduAsyncReqRow.java | 19 +++++++++---------- .../flink/sql/side/redis/RedisAllReqRow.java | 13 ++++++------- .../sql/sink/redis/RedisOutputFormat.java | 5 +---- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java index 9cb180f8e..db9b87d8d 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java @@ -54,7 +54,7 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 505b21479..31f9aa13d 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -7,11 +7,17 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 5d3d10a79..e534a85f9 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,20 +1,12 @@ package com.dtstack.flink.sql.side.kudu; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -24,6 +16,13 @@ import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 7bc4fe60e..02f480105 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -18,15 +18,10 @@ package com.dtstack.flink.sql.side.redis; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import com.esotericsoftware.minlog.Log; @@ -34,6 +29,10 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 8e807d0a9..ae4fe5a4b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -19,15 +19,12 @@ package com.dtstack.flink.sql.sink.redis; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; From d56226622c15ca0c3ece5b8103c6012dd0a0dde9 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 12 Mar 2020 10:41:46 +0800 Subject: [PATCH 027/378] fix conflict --- .../flink/sql/environment/StreamEnvConfigManager.java | 6 ++++++ hbase/pom.xml | 8 +------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index 2522437c5..d7b772c6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.environment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; diff --git a/hbase/pom.xml b/hbase/pom.xml index f64ffab1b..17f767215 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,13 +29,7 @@ 1.0-SNAPSHOT provided - - jdk.tools - jdk.tools - 1.6 - system - ${JAVA_HOME}/lib/tools.jar - + org.apache.hbase From ba2d191ccbb2444a824205ce413457dcc8aacccd Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 12 Mar 2020 18:42:35 +0800 Subject: [PATCH 028/378] rdb connect pool size --- .../clickhouse/ClickhouseAsyncReqRow.java | 6 ++--- .../flink/sql/side/AbstractSideTableInfo.java | 14 ++++++++++ .../flink/sql/side/BaseAsyncReqRow.java | 2 -- .../sql/table/AbstractSideTableParser.java | 9 +++++++ .../flink/sql/side/db2/Db2AsyncReqRow.java | 4 +-- hbase/pom.xml | 7 ----- .../sql/side/impala/ImpalaAsyncReqRow.java | 25 ++++++++--------- .../sql/side/mysql/MysqlAsyncReqRow.java | 9 ++----- .../sql/side/oracle/OracleAsyncReqRow.java | 9 ++----- .../sql/side/polardb/PolardbAsyncReqRow.java | 4 +-- .../postgresql/PostgresqlAsyncReqRow.java | 4 +-- .../side/rdb/all/AbstractRdbAllReqRow.java | 13 +++++++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 27 ++++++++++++++++++- .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +-- 14 files changed, 88 insertions(+), 49 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index e6f008d63..ec9a77c6b 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -48,14 +48,14 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS); System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setEventLoopPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index cacba74fd..0fe177f96 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -55,6 +55,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -65,6 +67,11 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private int asyncTimeout=10000; + /** + * async operator req outside conn pool size, egg rdb conn pool size + */ + private int asyncPoolSize = 0; + private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; @@ -156,4 +163,11 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } + public int getAsyncPoolSize() { + return asyncPoolSize; + } + + public void setAsyncPoolSize(int asyncPoolSize) { + this.asyncPoolSize = asyncPoolSize; + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index a0deb6b74..e5bce4437 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -30,14 +30,12 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 39b72c22f..4973a891e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.flink.util.Preconditions; + import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,6 +116,13 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); + sideTableInfo.setAsyncPoolSize(asyncPoolSize); + } + } } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 240bab9d1..8e7275977 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); db2lientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", DB2_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); diff --git a/hbase/pom.xml b/hbase/pom.xml index f64ffab1b..835457e29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,13 +29,6 @@ 1.0-SNAPSHOT provided - - jdk.tools - jdk.tools - 1.6 - system - ${JAVA_HOME}/lib/tools.jar - org.apache.hbase diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 77d3f3007..7ab603650 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -57,31 +57,28 @@ public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { @@ -61,7 +56,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", MYSQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -73,7 +68,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index c0b37e7ac..938fd870c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -30,16 +30,11 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; public class OracleAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(OracleAsyncReqRow.class); - private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { @@ -53,7 +48,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -65,7 +60,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 286f17286..13d25b111 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -55,7 +55,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POLARDB_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -67,7 +67,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 2a9e0137c..0efc077e0 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -60,13 +60,13 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); pgClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POSTGRESQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 17daecfb0..8fb35b7eb 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.rdb.all; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -73,6 +74,18 @@ public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + LOG.info("jdbc url: {} ", tableInfo.getUrl()); + LOG.info("query table name: {}", tableInfo.getTableName()); + LOG.info("all cache type: {}", tableInfo.getCacheType()); + LOG.info("all cache size: {}", tableInfo.getCacheSize()); + LOG.info("all cache timeout: {}", tableInfo.getCacheTimeout()); + } + + @Override protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index f78c30c30..b4a1c25e2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -24,11 +24,13 @@ import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -57,7 +59,9 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; @@ -71,6 +75,27 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); + init(sideInfo); + } + + protected void init(BaseSideInfo sideInfo) { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); + int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; + rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + LOG.info("use rdb pool size: {}", rdbSideTableInfo.getAsyncPoolSize()); + LOG.info("jdbc url: {} ", rdbSideTableInfo.getUrl()); + LOG.info("query table name: {}", rdbSideTableInfo.getTableName()); + LOG.info("async cache type: {}", rdbSideTableInfo.getCacheType()); + LOG.info("async cache mode: {}", rdbSideTableInfo.getCacheMode()); + LOG.info("async cache capacity: {}", rdbSideTableInfo.getAsyncCapacity()); + LOG.info("async cache timeout: {}", rdbSideTableInfo.getAsyncTimeout()); } @Override diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index f78061fca..a63d28607 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -55,7 +55,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -68,7 +68,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); From 229afced132eb5a367914b019daf08e4347e61e7 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 13 Mar 2020 11:12:56 +0800 Subject: [PATCH 029/378] connection info --- .../flink/sql/side/AbstractSideTableInfo.java | 15 +++++++++++++++ .../sql/side/rdb/all/AbstractRdbAllReqRow.java | 6 +----- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 8 +------- .../sql/side/rdb/table/RdbSideTableInfo.java | 11 +++++++++++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 0fe177f96..7a832d0a1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -170,4 +170,19 @@ public int getAsyncPoolSize() { public void setAsyncPoolSize(int asyncPoolSize) { this.asyncPoolSize = asyncPoolSize; } + + @Override + public String toString() { + return "Cache Info{" + + "cacheType='" + cacheType + '\'' + + ", cacheSize=" + cacheSize + + ", cacheTimeout=" + cacheTimeout + + ", asyncCapacity=" + asyncCapacity + + ", asyncTimeout=" + asyncTimeout + + ", asyncPoolSize=" + asyncPoolSize + + ", asyncTimeoutNumLimit=" + asyncTimeoutNumLimit + + ", partitionedJoin=" + partitionedJoin + + ", cacheMode='" + cacheMode + '\'' + + '}'; + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 8fb35b7eb..2e9986742 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -78,11 +78,7 @@ public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { public void open(Configuration parameters) throws Exception { super.open(parameters); RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("jdbc url: {} ", tableInfo.getUrl()); - LOG.info("query table name: {}", tableInfo.getTableName()); - LOG.info("all cache type: {}", tableInfo.getCacheType()); - LOG.info("all cache size: {}", tableInfo.getCacheSize()); - LOG.info("all cache timeout: {}", tableInfo.getCacheTimeout()); + LOG.info("rdb dim table config info: {} ", tableInfo.toString()); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b4a1c25e2..356e9d665 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -89,13 +89,7 @@ protected void init(BaseSideInfo sideInfo) { public void open(Configuration parameters) throws Exception { super.open(parameters); RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("use rdb pool size: {}", rdbSideTableInfo.getAsyncPoolSize()); - LOG.info("jdbc url: {} ", rdbSideTableInfo.getUrl()); - LOG.info("query table name: {}", rdbSideTableInfo.getTableName()); - LOG.info("async cache type: {}", rdbSideTableInfo.getCacheType()); - LOG.info("async cache mode: {}", rdbSideTableInfo.getCacheMode()); - LOG.info("async cache capacity: {}", rdbSideTableInfo.getAsyncCapacity()); - LOG.info("async cache timeout: {}", rdbSideTableInfo.getAsyncTimeout()); + LOG.info("rdb dim table config info: {} ", rdbSideTableInfo.toString()); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 506033007..9a4fa3c4f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -99,4 +99,15 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } + + @Override + public String toString() { + String cacheInfo = super.toString(); + String connectionInfo = "Rdb Side Connection Info{" + + "url='" + url + '\'' + + ", tableName='" + tableName + '\'' + + ", schema='" + schema + '\'' + + '}'; + return cacheInfo + " , " + connectionInfo; + } } From b3e9ea6b20efb8994e885fd76042827bdb3bb57b Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 16 Mar 2020 20:00:37 +0800 Subject: [PATCH 030/378] =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96=E5=8E=BB=E9=99=A4=E5=86=97=E4=BD=99=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/util/DateUtil.java | 410 ++++++++---------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 4 +- 2 files changed, 194 insertions(+), 220 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a0f5291e7..2b2dc4715 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -16,29 +16,24 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.Calendar; -import java.util.Date; -import java.util.Locale; -import java.util.SimpleTimeZone; -import java.util.TimeZone; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; /** - * + * * Reason: TODO ADD REASON(可选) * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com @@ -47,20 +42,16 @@ */ public class DateUtil { - static final String timeZone = "GMT+8"; - static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; - static final String dateFormat = "yyyy-MM-dd"; - static final String timeFormat = "HH:mm:ss"; - static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); - static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final int MILLIS_PER_SECOND = 1000; + public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { return new java.sql.Date(stringToDate((String)column).getTime()); @@ -84,23 +75,32 @@ public static Date stringToDate(String strDate) { return null; } try { - return datetimeFormatter.parse(strDate); - } catch (ParseException ignored) { + ; + return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); + } catch (DateTimeParseException ignored) { } try { - return dateFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); + } catch (DateTimeParseException ignored) { } try { - return timeFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); + } catch (DateTimeParseException ignored) { } throw new RuntimeException("can't parse date"); } + public static Date localDateTimetoDate(LocalDateTime localDateTime){ + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime dateToLocalDateTime(Date date){ + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + /** * * @@ -130,13 +130,13 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if(scope.equals("MS")){ - return getTodayStart(day)*1000; - }else if(scope.equals("S")){ - return getTodayStart(day); - }else{ - return getTodayStart(day); - } + if("MS".equals(scope)){ + return getTodayStart(day)*1000; + }else if("S".equals(scope)){ + return getTodayStart(day); + }else{ + return getTodayStart(day); + } } /** @@ -168,15 +168,15 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if(scope.equals("MS")){ - return getNextDayStart(day)*1000; - }else if(scope.equals("S")){ - return getNextDayStart(day); - }else{ - return getNextDayStart(day); - } + if("MS".equals(scope)){ + return getNextDayStart(day)*1000; + }else if("S".equals(scope)){ + return getNextDayStart(day); + }else{ + return getNextDayStart(day); + } } - + /** * @@ -237,7 +237,7 @@ public static long getWeekFirst(long day) { /** * 根据某个日期时间戳秒值，获取所在周在一年中是第几周. - * + * * @param day * @return */ @@ -257,17 +257,17 @@ public static int getWeekOfYear(long day) { */ public static String getYesterdayByString(String day, String inFormat, String outFormat){ try { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay - 1); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; - } catch (ParseException e) { - return null; - } + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay - 1); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; + } catch (ParseException e) { + return null; + } } /** @@ -288,7 +288,7 @@ public static String getTomorrowByString(String day, String inFormat, String out String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - + /** * * @param date @@ -320,7 +320,7 @@ public static String get30DaysBeforeByString(String day, String inFormat, String calendar.set(Calendar.DATE, calendarDay - 30); return new SimpleDateFormat(outFormat).format(calendar.getTime()); } - + /** * * @param day @@ -330,14 +330,14 @@ public static String get30DaysBeforeByString(String day, String inFormat, String * @throws ParseException */ public static String get30DaysLaterByString(String day, String inFormat, String outFormat) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay + 30); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay + 30); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; } @@ -349,7 +349,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -357,8 +357,8 @@ public static String getDateStrTOFormat(String day, String inFormat, String outF String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - - public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { + + public static long getDateMillToFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -474,79 +474,79 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MILLISECOND, 0); return cal.getTimeInMillis() / 1000; } - - /** - * @return long - */ + + /** + * @return long + */ public static long getMillByDay(int severalDays,String condition) { - int dateT=0; + int dateT=0; Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getMillToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getMillToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getMillToDay(cal,dateT); - } - return getMillToDay(cal,dateT); - } - + if(condition==null){ + return getMillToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getMillToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getMillToDay(cal,dateT); + } + return getMillToDay(cal,dateT); + } + /** * @return long */ public static long getStampByDay(int severalDays,String condition) { - int dateT=0; - Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getStampToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getStampToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getStampToDay(cal,dateT); - } - return getStampToDay(cal,dateT); + int dateT=0; + Calendar cal = Calendar.getInstance(); + if(condition==null){ + return getStampToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getStampToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getStampToDay(cal,dateT); + } + return getStampToDay(cal,dateT); } /** * @return long */ public static long getMillByDay(){ - return getMillByDay(0,null); + return getMillByDay(0,null); } - + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getMillToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; - } - + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis()/1000; + } + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getStampToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - return cal.getTimeInMillis(); + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + return cal.getTimeInMillis(); } public static String getToday() { @@ -568,7 +568,7 @@ public static String getDate(long day, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); } - + /** * * @param date @@ -578,21 +578,21 @@ public static String getDate(Date date, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(date); } - - + + /** * * @param day * @param format * @return long - * @throws ParseException + * @throws ParseException */ public static long stringToLong(String day, String format) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long Date = dateFormat.parse(day).getTime(); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + long date = dateFormat.parse(day).getTime(); + return date; } - + /** * @param day * @param format @@ -600,30 +600,30 @@ public static long stringToLong(String day, String format) throws ParseException * @throws ParseException */ public static Date stringToDate(String day, String format) { - try { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date Date = dateFormat.parse(day); - return Date; - } catch (ParseException e) { - return new Date(); - } + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + Date date = dateFormat.parse(day); + return date; + } catch (ParseException e) { + return new Date(); + } } - - + + /** * long型时间戳转为String型 - * + * * @param day 秒 * @return 格式化后的日期 - * @throws ParseException + * @throws ParseException */ public static String longToString(long day, String format) throws ParseException { - if (("" + day).length() <= 10){ + if (("" + day).length() <= 10){ day=day*1000; } - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String Date = dateFormat.format(day); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + String date = dateFormat.format(day); + return date; } /** @@ -653,39 +653,39 @@ public static long getMillByNow() { return cal.getTimeInMillis(); } - public static int getWeeksBetweenTwoDates(long startDay, long endDay) { - int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; - if(week<1){ - week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; - } - return week; - } + public static int getWeeksBetweenTwoDates(long startDay, long endDay) { + int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; + if(week<1){ + week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; + } + return week; + } - public static int getMaxWeekOfYear(long startDay) { - Calendar cal = Calendar.getInstance(); + public static int getMaxWeekOfYear(long startDay) { + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(startDay * 1000)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); - } - - public static int getMonthsBetweenTwoDates(long startDay, long endDay) { - int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; - if(month<1){ - month = getMonth(endDay) + 12 - getMonth(startDay) +1; - } - return month; - } - - public static Date parseDate(String dateStr, String pattern){ - SimpleDateFormat sdf = new SimpleDateFormat(); - sdf.applyPattern(pattern); - try { - return sdf.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - - /** + } + + public static int getMonthsBetweenTwoDates(long startDay, long endDay) { + int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; + if(month<1){ + month = getMonth(endDay) + 12 - getMonth(startDay) +1; + } + return month; + } + + public static Date parseDate(String dateStr, String pattern){ + SimpleDateFormat sdf = new SimpleDateFormat(); + sdf.applyPattern(pattern); + try { + return sdf.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + + /** * * @param time Long 时间 * @return long @@ -703,7 +703,7 @@ public static long getMinuteStart(long time) { firstDay = cal.getTimeInMillis() / 1000; return firstDay; } - + /** * @param time Long * @return long @@ -732,24 +732,24 @@ public static Date getDateByLong(long time){ date.setTime(time); return date; } - + public static Date parseDate(String dateStr, String pattern, Locale locale){ - SimpleDateFormat df = new SimpleDateFormat( - pattern, locale); - - df.setTimeZone(new SimpleTimeZone(0, "GMT")); - try { - return df.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - + SimpleDateFormat df = new SimpleDateFormat( + pattern, locale); + + df.setTimeZone(new SimpleTimeZone(0, "GMT")); + try { + return df.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + public static String getDate(Date date, String format, Locale locale) { - SimpleDateFormat df = new SimpleDateFormat( - format, locale); - df.setTimeZone(new SimpleTimeZone(0, "GMT")); + SimpleDateFormat df = new SimpleDateFormat( + format, locale); + df.setTimeZone(new SimpleTimeZone(0, "GMT")); return df.format(date); } @@ -777,59 +777,33 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - return dateFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_FORMATTER); } public static String timestampToString(Date date) { - return datetimeFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_TIME_FORMATTER); } - public static Timestamp getTimestampFromStr(String timeStr) { if (DATETIME.matcher(timeStr).matches()) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); - } else { - java.sql.Date date = null; - try { - date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("getTimestampFromStr error data is " + timeStr); - } - return new Timestamp(date.getTime()); } + return new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { - // 2020-01-01 format if (DATE.matcher(dateStr).matches()) { - // convert from local date to instant Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); - // calculate the timezone offset in millis int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); - // need to remove the offset since time has no TZ component return new java.sql.Date(instant.toEpochMilli() - offset); } else if (DATETIME.matcher(dateStr).matches()) { - // 2020-01-01T12:12:12Z format Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); - } else { - try { - // 2020-01-01 12:12:12.0 format - return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } } + return new java.sql.Date(stringToDate(dateStr).getTime()); } - - - public static String getStringFromTimestamp(Timestamp timestamp) { - return datetimeFormatter.format(timestamp); - } - - public static String getStringFromDate(java.sql.Date date) { - return dateFormatter.format(date); - } - + } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index cda1968fc..f48e6e296 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -178,9 +178,9 @@ private Object convertDataType(Object val) { } else if (val instanceof Instant) { } else if (val instanceof Timestamp) { - val = DateUtil.getStringFromTimestamp((Timestamp) val); + val = DateUtil.timestampToString((Timestamp) val); } else if (val instanceof java.util.Date) { - val = DateUtil.getStringFromDate((java.sql.Date) val); + val = DateUtil.dateToString((java.util.Date)val); } else { val = val.toString(); } From 14446f4dd844b1bb9da6c412e3556e135bafcba4 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 17 Mar 2020 16:22:07 +0800 Subject: [PATCH 031/378] remove hbase jdk tools --- hbase/pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index f64ffab1b..835457e29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -29,13 +29,6 @@ 1.0-SNAPSHOT provided - - jdk.tools - jdk.tools - 1.6 - system - ${JAVA_HOME}/lib/tools.jar - org.apache.hbase From ec2b84e7e1fa0c54aa7602923768b86671ab79b8 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 17 Mar 2020 20:49:15 +0800 Subject: [PATCH 032/378] kafk update mode --- .../format/SerializationMetricWrapper.java | 11 +- .../kafka/AbstractKafkaProducerFactory.java | 31 +- .../CustomerKeyedSerializationSchema.java | 35 +- .../AvroCRowSerializationSchema.java | 346 +++++++++++++++++ .../CsvCRowSerializationSchema.java | 349 ++++++++++++++++++ .../JsonCRowSerializationSchema.java | 234 ++++++++++++ .../sql/sink/kafka/table/KafkaSinkParser.java | 2 + .../sink/kafka/table/KafkaSinkTableInfo.java | 13 +- .../KafkaDeserializationMetricWrapper.java | 10 +- .../flink/sql/sink/kafka/KafkaProducer.java | 6 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 18 +- .../flink/sql/sink/kafka/KafkaProducer09.java | 5 +- .../sink/kafka/KafkaProducer09Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 21 +- .../sql/sink/kafka/KafkaProducer010.java | 6 +- .../sink/kafka/KafkaProducer010Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 21 +- .../sql/sink/kafka/KafkaProducer011.java | 6 +- .../sink/kafka/KafkaProducer011Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 20 +- .../source/kafka/KafkaConsumer011Factory.java | 2 +- .../flink/sql/source/kafka/KafkaSource.java | 4 +- 23 files changed, 1067 insertions(+), 86 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 3a5af18b1..8802198a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -24,6 +24,7 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; @@ -34,11 +35,11 @@ * author: toutian * create: 2019/12/24 */ -public class SerializationMetricWrapper implements SerializationSchema { +public class SerializationMetricWrapper implements SerializationSchema { private static final long serialVersionUID = 1L; - private SerializationSchema serializationSchema; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; @@ -47,7 +48,7 @@ public class SerializationMetricWrapper implements SerializationSchema { protected transient Meter dtNumRecordsOutRate; - public SerializationMetricWrapper(SerializationSchema serializationSchema) { + public SerializationMetricWrapper(SerializationSchema serializationSchema) { this.serializationSchema = serializationSchema; } @@ -57,7 +58,7 @@ public void initMetric() { } @Override - public byte[] serialize(Row element) { + public byte[] serialize(CRow element) { beforeSerialize(); byte[] row = serializationSchema.serialize(element); afterSerialize(); @@ -79,7 +80,7 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } - public SerializationSchema getSerializationSchema() { + public SerializationSchema getSerializationSchema() { return serializationSchema; } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 88c2ca939..ebd313b29 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -19,15 +19,18 @@ import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.AvroCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.CsvCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.AvroRowSerializationSchema; import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -51,42 +54,36 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); - protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); + protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = createSerializationSchema(kafkaSinkTableInfo, typeInformation); + return new SerializationMetricWrapper(serializationSchema); } - private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = null; + private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = null; if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + serializationSchema = new JsonCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonRowSerializationSchema(typeInformation); + serializationSchema = new JsonCRowSerializationSchema(typeInformation, kafkaSinkTableInfo.getUpdateMode()); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } - } else if (FormatType.CSV.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(typeInformation); + final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); serializationSchema = serSchemaBuilder.build(); - } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - - serializationSchema = new AvroRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); - + serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(),kafkaSinkTableInfo.getUpdateMode()); } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 498766564..cde8d1b1d 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,18 +2,20 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; -public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); @@ -30,38 +32,41 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization this.mapper = new ObjectMapper(); } - public byte[] serializeKey(Row element) { - if(partitionKeys == null || partitionKeys.length <=0){ + @Override + public byte[] serializeKey(CRow element) { + if (partitionKeys == null || partitionKeys.length <= 0) { return null; - } - SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); - if(serializationSchema instanceof JsonRowSerializationSchema){ - return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if (serializationSchema instanceof JsonCRowSerializationSchema) { + return serializeJsonKey((JsonCRowSerializationSchema) serializationSchema, element); } return null; } - public byte[] serializeValue(Row element) { + @Override + public byte[] serializeValue(CRow element) { return this.serializationMetricWrapper.serialize(element); } - public String getTargetTopic(Row element) { + @Override + public String getTargetTopic(CRow element) { return null; } - private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { + private byte[] serializeJsonKey(JsonCRowSerializationSchema jsonCRowSerializationSchema, CRow element) { try { - byte[] data = jsonRowSerializationSchema.serialize(element); + byte[] data = jsonCRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); StringBuilder sb = new StringBuilder(); - for(String key : partitionKeys){ - if(objectNode.has(key)){ + for (String key : partitionKeys) { + if (objectNode.has(key)) { sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); - } catch (Exception e){ - if(COUNTER.getAndIncrement() % 1000 == 0){ + } catch (Exception e) { + if (COUNTER.getAndIncrement() % 1000 == 0) { LOG.error("serializeJsonKey error", e); } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java new file mode 100644 index 000000000..34fa22c99 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.formats.avro.AvroRowDeserializationSchema; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +/** + * Serialization schema that serializes CROW into Avro bytes. + * + *

Serializes objects that are represented in (nested) Flink rows. It support types that + * are compatible with Flink's Table & SQL API. + * + *

Note: Changes in this class need to be kept in sync with the corresponding runtime + * class {@link AvroRowDeserializationSchema} and schema converter {@link AvroSchemaConverter}. + * + * @author maqi + */ +public class AvroCRowSerializationSchema implements SerializationSchema { + + /** + * Used for time conversions from SQL types. + */ + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + + /** + * Avro record class for serialization. Might be null if record class is not available. + */ + private Class recordClazz; + + /** + * Schema string for deserialization. + */ + private String schemaString; + + /** + * Avro serialization schema. + */ + private transient Schema schema; + + /** + * Writer to serialize Avro record into a byte array. + */ + private transient DatumWriter datumWriter; + + /** + * Output stream to serialize records into byte array. + */ + private transient ByteArrayOutputStream arrayOutputStream; + + /** + * Low-level class for serialization of Avro values. + */ + private transient Encoder encoder; + + private String updateMode; + + private final String retractKey = "retract"; + + /** + * Creates an Avro serialization schema for the given specific record class. + * + * @param recordClazz Avro record class used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(Class recordClazz) { + Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); + this.recordClazz = recordClazz; + this.schema = SpecificData.get().getSchema(recordClazz); + this.schemaString = schema.toString(); + this.datumWriter = new SpecificDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + } + + /** + * Creates an Avro serialization schema for the given Avro schema string. + * + * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(String avroSchemaString,String updateMode) { + Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); + this.recordClazz = null; + this.schemaString = avroSchemaString; + try { + this.schema = new Schema.Parser().parse(avroSchemaString); + } catch (SchemaParseException e) { + throw new IllegalArgumentException("Could not parse Avro schema string.", e); + } + this.datumWriter = new GenericDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; + } + + @Override + public byte[] serialize(CRow crow) { + try { + Row row = crow.row(); + boolean change = crow.change(); + + // convert to record + final GenericRecord record = convertRowToAvroRecord(schema, row); + arrayOutputStream.reset(); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + record.put(retractKey, change); + } + datumWriter.write(record, encoder); + encoder.flush(); + return arrayOutputStream.toByteArray(); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize row.", e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AvroCRowSerializationSchema that = (AvroCRowSerializationSchema) o; + return Objects.equals(recordClazz, that.recordClazz) && Objects.equals(schemaString, that.schemaString); + } + + @Override + public int hashCode() { + return Objects.hash(recordClazz, schemaString); + } + + // -------------------------------------------------------------------------------------------- + + private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { + final List fields = schema.getFields(); + final int length = fields.size(); + final GenericRecord record = new GenericData.Record(schema); + for (int i = 0; i < length; i++) { + final Schema.Field field = fields.get(i); + record.put(i, convertFlinkType(field.schema(), row.getField(i))); + } + return record; + } + + private Object convertFlinkType(Schema schema, Object object) { + if (object == null) { + return null; + } + switch (schema.getType()) { + case RECORD: + if (object instanceof Row) { + return convertRowToAvroRecord(schema, (Row) object); + } + throw new IllegalStateException("Row expected but was: " + object.getClass()); + case ENUM: + return new GenericData.EnumSymbol(schema, object.toString()); + case ARRAY: + final Schema elementSchema = schema.getElementType(); + final Object[] array = (Object[]) object; + final GenericData.Array convertedArray = new GenericData.Array<>(array.length, schema); + for (Object element : array) { + convertedArray.add(convertFlinkType(elementSchema, element)); + } + return convertedArray; + case MAP: + final Map map = (Map) object; + final Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + convertedMap.put( + new Utf8(entry.getKey().toString()), + convertFlinkType(schema.getValueType(), entry.getValue())); + } + return convertedMap; + case UNION: + final List types = schema.getTypes(); + final int size = types.size(); + final Schema actualSchema; + if (size == 2 && types.get(0).getType() == Schema.Type.NULL) { + actualSchema = types.get(1); + } else if (size == 2 && types.get(1).getType() == Schema.Type.NULL) { + actualSchema = types.get(0); + } else if (size == 1) { + actualSchema = types.get(0); + } else { + // generic type + return object; + } + return convertFlinkType(actualSchema, object); + case FIXED: + // check for logical type + if (object instanceof BigDecimal) { + return new GenericData.Fixed( + schema, + convertFromDecimal(schema, (BigDecimal) object)); + } + return new GenericData.Fixed(schema, (byte[]) object); + case STRING: + return new Utf8(object.toString()); + case BYTES: + // check for logical type + if (object instanceof BigDecimal) { + return ByteBuffer.wrap(convertFromDecimal(schema, (BigDecimal) object)); + } + return ByteBuffer.wrap((byte[]) object); + case INT: + // check for logical types + if (object instanceof Date) { + return convertFromDate(schema, (Date) object); + } else if (object instanceof Time) { + return convertFromTime(schema, (Time) object); + } + return object; + case LONG: + // check for logical type + if (object instanceof Timestamp) { + return convertFromTimestamp(schema, (Timestamp) object); + } + return object; + case FLOAT: + case DOUBLE: + case BOOLEAN: + return object; + } + throw new RuntimeException("Unsupported Avro type:" + schema); + } + + private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType instanceof LogicalTypes.Decimal) { + final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; + // rescale to target type + final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); + // byte array must contain the two's-complement representation of the + // unscaled integer value in big-endian byte order + return decimal.unscaledValue().toByteArray(); + } else { + throw new RuntimeException("Unsupported decimal type."); + } + } + + private int convertFromDate(Schema schema, Date date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.date()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted / 86400000L); + } else { + throw new RuntimeException("Unsupported date type."); + } + } + + private int convertFromTime(Schema schema, Time date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timeMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted % 86400000L); + } else { + throw new RuntimeException("Unsupported time type."); + } + } + + private long convertFromTimestamp(Schema schema, Timestamp date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timestampMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + return time + (long) LOCAL_TZ.getOffset(time); + } else { + throw new RuntimeException("Unsupported timestamp type."); + } + } + + private void writeObject(ObjectOutputStream outputStream) throws IOException { + outputStream.writeObject(recordClazz); + outputStream.writeObject(schemaString); // support for null + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { + recordClazz = (Class) inputStream.readObject(); + schemaString = (String) inputStream.readObject(); + if (recordClazz != null) { + schema = SpecificData.get().getSchema(recordClazz); + } else { + schema = new Schema.Parser().parse(schemaString); + } + datumWriter = new SpecificDatumWriter<>(schema); + arrayOutputStream = new ByteArrayOutputStream(); + encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java new file mode 100644 index 000000000..903395f9d --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.csv.CsvRowDeserializationSchema; +import org.apache.flink.formats.csv.CsvRowSchemaConverter; +import org.apache.flink.formats.csv.CsvRowSerializationSchema; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Objects; + +/** + * Serialization schema that serializes an object of Flink types into a CSV bytes. + * + *

Serializes the input row into a {@link ObjectNode} and + * converts it into byte[]. + * + *

Result byte[] messages can be deserialized using {@link CsvRowDeserializationSchema}. + */ +@PublicEvolving +public final class CsvCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = 2098447220136965L; + + /** Type information describing the input CSV data. */ + private final RowTypeInfo typeInfo; + + /** Runtime instance that performs the actual work. */ + private final RuntimeConverter runtimeConverter; + + /** CsvMapper used to write {@link JsonNode} into bytes. */ + private final CsvMapper csvMapper; + + /** Schema describing the input CSV data. */ + private final CsvSchema csvSchema; + + /** Object writer used to write rows. It is configured by {@link CsvSchema}. */ + private final ObjectWriter objectWriter; + + /** Reusable object node. */ + private transient ObjectNode root; + + private String updateMode; + + private final String retractKey = "retract"; + + private CsvCRowSerializationSchema( + RowTypeInfo typeInfo, + CsvSchema csvSchema) { + this.typeInfo = typeInfo; + this.runtimeConverter = createRowRuntimeConverter(typeInfo, true); + this.csvMapper = new CsvMapper(); + this.csvSchema = csvSchema; + this.objectWriter = csvMapper.writer(csvSchema); + } + + /** + * A builder for creating a {@link CsvRowSerializationSchema}. + */ + @PublicEvolving + public static class Builder { + + private final RowTypeInfo typeInfo; + private CsvSchema csvSchema; + + /** + * Creates a {@link CsvRowSerializationSchema} expecting the given {@link TypeInformation}. + * + * @param typeInfo type information used to create schema. + */ + public Builder(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information must not be null."); + + if (!(typeInfo instanceof CRowTypeInfo)) { + throw new IllegalArgumentException("Row type information expected."); + } + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + this.typeInfo = rowTypeInfo; + this.csvSchema = CsvRowSchemaConverter.convert(rowTypeInfo); + } + + public Builder setFieldDelimiter(char c) { + this.csvSchema = this.csvSchema.rebuild().setColumnSeparator(c).build(); + return this; + } + + public Builder setLineDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + throw new IllegalArgumentException( + "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + } + this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); + return this; + } + + public Builder setArrayElementDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + this.csvSchema = this.csvSchema.rebuild().setArrayElementSeparator(delimiter).build(); + return this; + } + + public Builder setQuoteCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setQuoteChar(c).build(); + return this; + } + + public Builder setEscapeCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setEscapeChar(c).build(); + return this; + } + + public Builder setNullLiteral(String s) { + this.csvSchema = this.csvSchema.rebuild().setNullValue(s).build(); + return this; + } + + public CsvCRowSerializationSchema build() { + return new CsvCRowSerializationSchema( + typeInfo, + csvSchema); + } + } + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (root == null) { + root = csvMapper.createObjectNode(); + } + try { + runtimeConverter.convert(csvMapper, root, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + root.put(retractKey, change); + } + return objectWriter.writeValueAsBytes(root); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'.", t); + } + } + + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != this.getClass()) { + return false; + } + if (this == o) { + return true; + } + final CsvCRowSerializationSchema that = (CsvCRowSerializationSchema) o; + final CsvSchema otherSchema = that.csvSchema; + + return typeInfo.equals(that.typeInfo) && + csvSchema.getColumnSeparator() == otherSchema.getColumnSeparator() && + Arrays.equals(csvSchema.getLineSeparator(), otherSchema.getLineSeparator()) && + csvSchema.getArrayElementSeparator().equals(otherSchema.getArrayElementSeparator()) && + csvSchema.getQuoteChar() == otherSchema.getQuoteChar() && + csvSchema.getEscapeChar() == otherSchema.getEscapeChar() && + Arrays.equals(csvSchema.getNullValue(), otherSchema.getNullValue()); + } + + @Override + public int hashCode() { + return Objects.hash( + typeInfo, + csvSchema.getColumnSeparator(), + csvSchema.getLineSeparator(), + csvSchema.getArrayElementSeparator(), + csvSchema.getQuoteChar(), + csvSchema.getEscapeChar(), + csvSchema.getNullValue()); + } + + // -------------------------------------------------------------------------------------------- + + private interface RuntimeConverter extends Serializable { + JsonNode convert(CsvMapper csvMapper, ContainerNode container, Object obj); + } + + private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) { + final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes(); + final String[] fieldNames = rowTypeInfo.getFieldNames(); + + final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes); + + return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters); + } + + private static RuntimeConverter[] createFieldRuntimeConverters(TypeInformation[] fieldTypes) { + final RuntimeConverter[] fieldConverters = new RuntimeConverter[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + fieldConverters[i] = createNullableRuntimeConverter(fieldTypes[i]); + } + return fieldConverters; + } + + private static RuntimeConverter assembleRowRuntimeConverter( + boolean isTopLevel, + String[] fieldNames, + RuntimeConverter[] fieldConverters) { + final int rowArity = fieldNames.length; + // top level reuses the object node container + if (isTopLevel) { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ObjectNode objectNode = (ObjectNode) container; + for (int i = 0; i < rowArity; i++) { + objectNode.set( + fieldNames[i], + fieldConverters[i].convert(csvMapper, container, row.getField(i))); + } + return objectNode; + }; + } else { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (int i = 0; i < rowArity; i++) { + arrayNode.add(fieldConverters[i].convert(csvMapper, arrayNode, row.getField(i))); + } + return arrayNode; + }; + } + } + + private static RuntimeConverter createNullableRuntimeConverter(TypeInformation info) { + final RuntimeConverter valueConverter = createRuntimeConverter(info); + return (csvMapper, container, obj) -> { + if (obj == null) { + return container.nullNode(); + } + return valueConverter.convert(csvMapper, container, obj); + }; + } + + private static RuntimeConverter createRuntimeConverter(TypeInformation info) { + if (info.equals(Types.VOID)) { + return (csvMapper, container, obj) -> container.nullNode(); + } else if (info.equals(Types.STRING)) { + return (csvMapper, container, obj) -> container.textNode((String) obj); + } else if (info.equals(Types.BOOLEAN)) { + return (csvMapper, container, obj) -> container.booleanNode((Boolean) obj); + } else if (info.equals(Types.BYTE)) { + return (csvMapper, container, obj) -> container.numberNode((Byte) obj); + } else if (info.equals(Types.SHORT)) { + return (csvMapper, container, obj) -> container.numberNode((Short) obj); + } else if (info.equals(Types.INT)) { + return (csvMapper, container, obj) -> container.numberNode((Integer) obj); + } else if (info.equals(Types.LONG)) { + return (csvMapper, container, obj) -> container.numberNode((Long) obj); + } else if (info.equals(Types.FLOAT)) { + return (csvMapper, container, obj) -> container.numberNode((Float) obj); + } else if (info.equals(Types.DOUBLE)) { + return (csvMapper, container, obj) -> container.numberNode((Double) obj); + } else if (info.equals(Types.BIG_DEC)) { + return (csvMapper, container, obj) -> container.numberNode((BigDecimal) obj); + } else if (info.equals(Types.BIG_INT)) { + return (csvMapper, container, obj) -> container.numberNode((BigInteger) obj); + } else if (info.equals(Types.SQL_DATE)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIME)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info instanceof RowTypeInfo){ + return createRowRuntimeConverter((RowTypeInfo) info, false); + } else if (info instanceof BasicArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((BasicArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof ObjectArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((ObjectArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof PrimitiveArrayTypeInfo && + ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return createByteArrayRuntimeConverter(); + } + else { + throw new RuntimeException("Unsupported type information '" + info + "'."); + } + } + + private static RuntimeConverter createObjectArrayRuntimeConverter(TypeInformation elementType) { + final RuntimeConverter elementConverter = createNullableRuntimeConverter(elementType); + return (csvMapper, container, obj) -> { + final Object[] array = (Object[]) obj; + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (Object element : array) { + arrayNode.add(elementConverter.convert(csvMapper, arrayNode, element)); + } + return arrayNode; + }; + } + + private static RuntimeConverter createByteArrayRuntimeConverter() { + return (csvMapper, container, obj) -> container.binaryNode((byte[]) obj); + } + + private static void validateArity(int expected, int actual) { + if (expected != actual) { + throw new RuntimeException("Row length mismatch. " + expected + + " fields expected but was " + actual + "."); + } + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java new file mode 100644 index 000000000..bfe801d52 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.Objects; + +/** + * + * Serialization schema that serializes an object of Flink types into a JSON bytes. + * + *

Serializes the input Flink object into a JSON string and + * converts it into byte[]. + * + */ +public class JsonCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = -2885556750743978636L; + + /** Type information describing the input type. */ + private final TypeInformation typeInfo; + + /** Object mapper that is used to create output JSON objects. */ + private final ObjectMapper mapper = new ObjectMapper(); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ + private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ + private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ + private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** Reusable object node. */ + private transient ObjectNode node; + + private String updateMode; + + private final String retractKey = "retract"; + + public JsonCRowSerializationSchema(String jsonSchema, String updateMode) { + this(JsonRowSchemaConverter.convert(jsonSchema), updateMode); + } + + /** + * Creates a JSON serialization schema for the given type information. + * + * @param typeInfo The field names of {@link Row} are used to map to JSON properties. + */ + public JsonCRowSerializationSchema(TypeInformation typeInfo, String updateMode) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + this.updateMode = updateMode; + } + + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (node == null) { + node = mapper.createObjectNode(); + } + + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + try { + convertRow(node, rowTypeInfo, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + node.put(retractKey, change); + } + return mapper.writeValueAsBytes(node); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'. " + + "Make sure that the schema matches the input.", t); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final JsonCRowSerializationSchema that = (JsonCRowSerializationSchema) o; + return Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo); + } + + // -------------------------------------------------------------------------------------------- + + private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { + if (reuse == null) { + reuse = mapper.createObjectNode(); + } + final String[] fieldNames = info.getFieldNames(); + + final TypeInformation[] fieldTypes = info.getFieldTypes(); + + // validate the row + if (row.getArity() != fieldNames.length) { + throw new IllegalStateException(String.format( + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + } + + for (int i = 0; i < fieldNames.length; i++) { + final String name = fieldNames[i]; + + final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); + reuse.set(name, fieldConverted); + } + + return reuse; + } + + private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { + if (info == Types.VOID || object == null) { + return container.nullNode(); + } else if (info == Types.BOOLEAN) { + return container.booleanNode((Boolean) object); + } else if (info == Types.STRING) { + return container.textNode((String) object); + } else if (info == Types.BIG_DEC) { + // convert decimal if necessary + if (object instanceof BigDecimal) { + return container.numberNode((BigDecimal) object); + } + return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); + } else if (info == Types.BIG_INT) { + // convert integer if necessary + if (object instanceof BigInteger) { + return container.numberNode((BigInteger) object); + } + return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); + } else if (info == Types.SQL_DATE) { + return container.textNode(object.toString()); + } else if (info == Types.SQL_TIME) { + final Time time = (Time) object; + // strip milliseconds if possible + if (time.getTime() % 1000 > 0) { + return container.textNode(timeFormatWithMillis.format(time)); + } + return container.textNode(timeFormat.format(time)); + } else if (info == Types.SQL_TIMESTAMP) { + return container.textNode(timestampFormat.format((Timestamp) object)); + } else if (info instanceof RowTypeInfo) { + if (reuse != null && reuse instanceof ObjectNode) { + return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); + } else { + return convertRow(null, (RowTypeInfo) info, (Row) object); + } + } else if (info instanceof ObjectArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof BasicArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return container.binaryNode((byte[]) object); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return mapper.valueToTree(object); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); + } + } + } + + private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { + if (reuse == null) { + reuse = mapper.createArrayNode(); + } else { + reuse.removeAll(); + } + + for (Object object : array) { + reuse.add(convert(reuse, null, info, object)); + } + return reuse; + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 7520de512..a42473420 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.kafka.table; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -51,6 +52,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -58,6 +60,8 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String partitionKeys; + private String updateMode; + public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -70,7 +74,6 @@ public Set getKafkaParamKeys() { return kafkaParam.keySet(); } - public String getBootstrapServers() { return bootstrapServers; } @@ -103,6 +106,14 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } + public String getUpdateMode() { + return updateMode; + } + + public void setUpdateMode(String updateMode) { + this.updateMode = updateMode; + } + @Override public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index afa950c5b..f08287cec 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -77,7 +77,7 @@ protected void beforeDeserialize() throws IOException { protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + Field consumerThreadField = getConsumerThreadField(fetcher); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); @@ -115,6 +115,14 @@ public Long getValue() { } } + private Field getConsumerThreadField(AbstractFetcher fetcher) throws NoSuchFieldException { + try { + return fetcher.getClass().getDeclaredField("consumerThread"); + } catch (Exception e) { + return fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + } + } + public void setFetcher(AbstractFetcher fetcher) { this.fetcher = fetcher; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 71a9cc386..1cbbeafd9 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer extends FlinkKafkaProducer { +public class KafkaProducer extends FlinkKafkaProducer { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); @@ -45,7 +45,7 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index f3a2f40f5..6bf9014df 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.util.Optional; import java.util.Properties; @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 7105bc037..71e938ba5 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -31,6 +31,8 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -56,13 +58,15 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Properties properties; - protected FlinkKafkaProducer flinkKafkaProducer; + protected FlinkKafkaProducer flinkKafkaProducer; + protected CRowTypeInfo typeInformation; + /** The schema of the table. */ private TableSchema schema; /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; + protected Optional> partitioner; private String[] partitionKeys; @@ -97,7 +101,9 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.flinkKafkaProducer = (FlinkKafkaProducer) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + this.flinkKafkaProducer = (FlinkKafkaProducer) new KafkaProducerFactory() + .createKafkaProducer(kafkaSinkTableInfo, typeInformation, properties, partitioner, partitionKeys); return this; } @@ -108,9 +114,9 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) .setParallelism(parallelism); mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index c815e134a..bee1865dd 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer09 extends FlinkKafkaProducer09 { +public class KafkaProducer09 extends FlinkKafkaProducer09 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); @@ -45,7 +46,7 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index 7fb3909ee..ee3423b07 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 0a991a8ea..e6dbdf3d3 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -27,10 +27,13 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -56,13 +59,14 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Properties properties; - protected FlinkKafkaProducer09 kafkaProducer09; + protected FlinkKafkaProducer09 kafkaProducer09; + protected CRowTypeInfo typeInformation; /** The schema of the table. */ private TableSchema schema; /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; + protected Optional> partitioner; private String[] partitionKeys; @@ -101,8 +105,9 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.kafkaProducer09 = (FlinkKafkaProducer09) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + this.kafkaProducer09 = (FlinkKafkaProducer09) new KafkaProducer09Factory() + .createKafkaProducer(kafka09SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); return this; } @@ -113,10 +118,10 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) + .setParallelism(parallelism); mapDataStream.addSink(kafkaProducer09) .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3cdc45dec..3936575ef 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer010 extends FlinkKafkaProducer010 { +public class KafkaProducer010 extends FlinkKafkaProducer010 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); @@ -45,7 +45,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index c44a9fe86..e0e023586 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 34ea8fc5f..ac5a11810 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -29,6 +29,8 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -61,6 +63,9 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected KafkaSinkTableInfo kafka10SinkTableInfo; + protected RichSinkFunction kafkaProducer010; + protected CRowTypeInfo typeInformation; + /** The schema of the table. */ private TableSchema schema; @@ -97,6 +102,12 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { if (parallelism != null) { this.parallelism = parallelism; } + + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, + typeInformation, properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); + return this; } @@ -107,13 +118,9 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - - RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition<>()), partitionKeys); - - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) .setParallelism(parallelism); mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 7880dd419..429d21a79 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer011 extends FlinkKafkaProducer011 { +public class KafkaProducer011 extends FlinkKafkaProducer011 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); @@ -46,7 +46,7 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index e2272b16e..0cb11da82 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d7807a935..835941ca3 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -31,6 +31,8 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.table.utils.TableConnectorUtils; @@ -61,13 +63,15 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener protected Properties properties; - protected FlinkKafkaProducer011 kafkaProducer011; + protected FlinkKafkaProducer011 kafkaProducer011; + protected CRowTypeInfo typeInformation; + /** The schema of the table. */ private TableSchema schema; /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; + protected Optional> partitioner; private String[] partitionKeys; @@ -102,8 +106,9 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.parallelism = parallelism; } - this.kafkaProducer011 = (FlinkKafkaProducer011) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + typeInformation = new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + this.kafkaProducer011 = (FlinkKafkaProducer011) new KafkaProducer011Factory() + .createKafkaProducer(kafka11SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); return this; } @@ -114,9 +119,10 @@ public TypeInformation getRecordType() { @Override public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) + + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(typeInformation) .setParallelism(parallelism); mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java index 3f804fc6c..c20f0678b 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java @@ -31,8 +31,8 @@ /** * company: www.dtstack.com - * author: toutian * create: 2019/12/24 + * @author: toutian */ public class KafkaConsumer011Factory extends AbstractKafkaConsumerFactory { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 11be1898a..2f760bdf2 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -93,11 +93,11 @@ public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExec //earliest,latest if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { try { Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); + Map specificStartupOffsets = new HashMap<>(16); for (Map.Entry entry : offsetMap.entrySet()) { specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); } From f683c37c78758926225333c451adf943bc6a3e70 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 18 Mar 2020 15:56:31 +0800 Subject: [PATCH 033/378] =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E5=BC=82=E6=AD=A5cac?= =?UTF-8?q?he=E7=9A=84=E6=8A=BD=E8=B1=A1=E5=92=8C=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/cassandra/CassandraAsyncReqRow.java | 87 ++++--------- .../flink/sql/side/AbstractSideTableInfo.java | 13 ++ .../flink/sql/side/BaseAsyncReqRow.java | 114 +++++++++++++++++- .../sql/table/AbstractSideTableParser.java | 7 ++ .../Elasticsearch6AsyncReqRow.java | 63 +++------- .../sql/side/hbase/HbaseAsyncReqRow.java | 50 +------- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 77 +++--------- .../sql/side/mongo/MongoAsyncReqRow.java | 56 +++------ 8 files changed, 210 insertions(+), 257 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index e1ed6d8e1..319e82ff3 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -162,61 +162,13 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); - JsonArray inputParams = new JsonArray(); - StringBuffer stringBuffer = new StringBuffer(); - String sqlWhere = " where "; - - for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(inputCopy, resultFuture); - return; - } - inputParams.add(equalObj); - StringBuffer sqlTemp = stringBuffer.append(sideInfo.getEqualFieldList().get(i)) - .append(" = "); - if (equalObj instanceof String) { - sqlTemp.append("'" + equalObj + "'") - .append(" and "); - } else { - sqlTemp.append(equalObj) - .append(" and "); - } - - } + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); - sqlWhere = sqlWhere + stringBuffer.toString().substring(0, stringBuffer.lastIndexOf(" and ")); - - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), jsonArray); - rowList.add(new CRow(row, inputCopy.change())); - } - resultFuture.complete(rowList); - } else { - throw new RuntimeException("not support cache obj type " + val.getType()); - } - return; - } - } - //connect Cassandra connCassandraDB(cassandraSideTableInfo); - String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - System.out.println("sqlCondition:" + sqlCondition); + String sqlCondition = sideInfo.getSqlCondition() + " " + buildWhereCondition(inputParams) + " ALLOW FILTERING "; ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -242,18 +194,18 @@ public void onSuccess(List rows) { List cacheContent = Lists.newArrayList(); List rowList = Lists.newArrayList(); for (com.datastax.driver.core.Row line : rows) { - Row row = fillData(inputCopy.row(), line); + Row row = fillData(input.row(), line); if (openCache()) { cacheContent.add(line); } - rowList.add(new CRow(row,inputCopy.change())); + rowList.add(new CRow(row, input.change())); } resultFuture.complete(rowList); if (openCache()) { putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } } else { - dealMissKey(inputCopy, resultFuture); + dealMissKey(input, resultFuture); if (openCache()) { putCache(key, CacheMissVal.getMissKeyObj()); } @@ -265,13 +217,30 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); - System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } }); } + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()).append("_"); + } + return sb.toString(); + } + + private String buildWhereCondition(Map inputParams){ + StringBuilder sb = new StringBuilder(" where "); + for(Map.Entry entry : inputParams.entrySet()){ + Object value = entry.getValue() instanceof String ? "'" + entry.getValue() + "'" : entry.getValue(); + sb.append(String.format("%s = %s", entry.getKey(), value)); + } + return sb.toString(); + } + @Override public Row fillData(Row input, Object line) { com.datastax.driver.core.Row rowArray = (com.datastax.driver.core.Row) line; @@ -306,14 +275,4 @@ public void close() throws Exception { cluster = null; } } - - public String buildCacheKey(JsonArray jsonArray) { - StringBuilder sb = new StringBuilder(); - for (Object ele : jsonArray.getList()) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index cacba74fd..0a6babc8a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -27,6 +27,7 @@ import java.io.Serializable; import java.util.List; +import java.util.Objects; /** * Reason: @@ -55,6 +56,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + public static final String ASYNC_FAIL_MAX_NUM_KEY = "asyncFailMaxNum"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -71,6 +74,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private String cacheMode="ordered"; + private Integer asyncFailMaxNum; + private List predicateInfoes = Lists.newArrayList(); public RowTypeInfo getRowTypeInfo(){ @@ -156,4 +161,12 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } + public Integer getAsyncFailMaxNum(Integer defaultValue) { + return Objects.isNull(asyncFailMaxNum) ? defaultValue : asyncFailMaxNum; + } + + public void setAsyncFailMaxNum(Integer asyncFailMaxNum) { + this.asyncFailMaxNum = asyncFailMaxNum; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index a0deb6b74..ae987e142 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -20,24 +20,36 @@ package com.dtstack.flink.sql.side; +import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; +import org.apache.commons.collections.MapUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.concurrent.TimeoutException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -126,7 +138,6 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } - timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); @@ -135,6 +146,105 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio } } + protected void preInvoke(CRow input, ResultFuture resultFuture){ + ScheduledFuture timeFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timeFuture); + } + + @Override + public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + preInvoke(input, resultFuture); + Map inputParams = parseInputParam(input); + if(MapUtils.isEmpty(inputParams)){ + dealMissKey(input, resultFuture); + return; + } + if(isUseCache(inputParams)){ + invokeWithCache(inputParams, input, resultFuture); + return; + } + handleAsyncInvoke(inputParams, input, resultFuture); + } + + private Map parseInputParam(CRow input){ + Map inputParams = Maps.newHashMap(); + for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { + Integer conValIndex = sideInfo.getEqualValIndex().get(i); + Object equalObj = input.row().getField(conValIndex); + if(equalObj == null){ + return inputParams; + } + String columnName = sideInfo.getEqualFieldList().get(conValIndex); + inputParams.put(columnName, equalObj.toString()); + } + return inputParams; + } + protected boolean isUseCache(Map inputParams){ + return openCache() && getFromCache(buildCacheKey(inputParams)) != null; + } + + private void invokeWithCache(Map inputParams, CRow input, ResultFuture resultFuture){ + if (openCache()) { + CacheObj val = getFromCache(buildCacheKey(inputParams)); + if (val != null) { + if (ECacheContentType.MissVal == val.getType()) { + dealMissKey(input, resultFuture); + return; + }else if(ECacheContentType.SingleLine == val.getType()){ + try { + Row row = fillData(input.row(), val); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input); + } + } else if (ECacheContentType.MultiLine == val.getType()) { + try { + List rowList = Lists.newArrayList(); + for (Object one : (List) val.getContent()) { + Row row = fillData(input.row(), one); + rowList.add(new CRow(row, input.change())); + } + resultFuture.complete(rowList); + } catch (Exception e) { + dealFillDataError(resultFuture, e, input); + } + } else { + resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); + } + return; + } + } + } + + public abstract void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception; + + public abstract String buildCacheKey(Map inputParams); + + private ProcessingTimeService getProcessingTimeService(){ + return ((StreamingRuntimeContext)this.getRuntimeContext()).getProcessingTimeService(); + } + + protected ScheduledFuture registerTimer(CRow input, ResultFuture resultFuture){ + long timeoutTimestamp = sideInfo.getSideTableInfo().getAsyncTimeout() + getProcessingTimeService().getCurrentProcessingTime(); + return getProcessingTimeService().registerTimer( + timeoutTimestamp, + new ProcessingTimeCallback() { + @Override + public void onProcessingTime(long timestamp) throws Exception { + timeout(input, resultFuture); + } + }); + } + + protected void cancelTimerWhenComplete(ResultFuture resultFuture, ScheduledFuture timerFuture){ + ThreadPoolExecutor executors = new ThreadPoolExecutor(1, 1,0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()); + if(resultFuture instanceof StreamRecordQueueEntry){ + StreamRecordQueueEntry streamRecordBufferEntry = (StreamRecordQueueEntry) resultFuture; + streamRecordBufferEntry.onComplete((Object value) -> { + timerFuture.cancel(true); + },executors); + } + } protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData) { LOG.debug("source data {} join side table error ", sourceData); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 39b72c22f..83a06d256 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -114,6 +114,13 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ + sideTableInfo.setAsyncFailMaxNum(asyncFailNum); + } + } } } } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index e8164edb2..f8eb66860 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -81,39 +81,8 @@ public void open(Configuration parameters) throws Exception { @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow copyCrow = new CRow(input.row(), input.change()); - List inputParams = Lists.newArrayList(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = copyCrow.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(copyCrow, resultFuture); - return; - } - inputParams.add(equalObj); - } - + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { String key = buildCacheKey(inputParams); - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(copyCrow, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - List rowList = getRows(copyCrow, null, (List) val.getContent()); - resultFuture.complete(rowList); - } catch (Exception e) { - dealFillDataError(resultFuture, e, copyCrow); - } - } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); - } - return; - } - } - BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); SearchSourceBuilder searchSourceBuilder = initConfiguration(); @@ -135,7 +104,7 @@ public void onResponse(SearchResponse searchResponse) { RestHighLevelClient tmpRhlClient = null; try { while (true) { - loadDataToCache(searchHits, rowList, cacheContent, copyCrow); + loadDataToCache(searchHits, rowList, cacheContent, input); // determine if all results haven been ferched if (searchHits.length < getFetchSize()) { break; @@ -154,7 +123,7 @@ public void onResponse(SearchResponse searchResponse) { dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, copyCrow); + dealFillDataError(resultFuture, e, input); } finally { if (tmpRhlClient != null) { try { @@ -165,7 +134,7 @@ public void onResponse(SearchResponse searchResponse) { } } } else { - dealMissKey(copyCrow, resultFuture); + dealMissKey(input, resultFuture); dealCacheData(key, CacheMissVal.getMissKeyObj()); } } @@ -177,7 +146,17 @@ public void onFailure(Exception e) { resultFuture.completeExceptionally(new RuntimeException("Response failed!")); } }); + } + + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()) + .append("_"); + } + return sb.toString(); } private void loadDataToCache(SearchHit[] searchHits, List rowList, List cacheContent, CRow copyCrow) { @@ -238,16 +217,6 @@ public void close() throws Exception { } - public String buildCacheKey(List equalValList) { - StringBuilder sb = new StringBuilder(); - for (Object ele : equalValList) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - private SearchSourceBuilder initConfiguration() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(getFetchSize()); @@ -258,7 +227,7 @@ private SearchSourceBuilder initConfiguration() { return searchSourceBuilder; } - private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuilder boolQueryBuilder) { + private BoolQueryBuilder setInputParams(Map inputParams, BoolQueryBuilder boolQueryBuilder) { if (boolQueryBuilder == null) { boolQueryBuilder = new BoolQueryBuilder(); } @@ -266,7 +235,7 @@ private BoolQueryBuilder setInputParams(List inputParams, BoolQueryBuild for (int i = 0; i < sqlJoinCompareOperate.size(); i++) { String fieldName = sideInfo.getEqualFieldList().get(i); String operatorKind = sqlJoinCompareOperate.get(sideInfo.getEqualFieldList().indexOf(fieldName)); - String condition = String.valueOf(inputParams.get(i)); + String condition = String.valueOf(inputParams.get(fieldName)); boolQueryBuilder = Es6Util.buildFilterCondition(boolQueryBuilder, new PredicateInfo(null, operatorKind, null, fieldName, condition), sideInfo); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 24ee01786..007de8e15 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -123,55 +123,17 @@ public void open(Configuration parameters) throws Exception { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); - Map refData = Maps.newHashMap(); - for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if(equalObj == null){ - dealMissKey(inputCopy, resultFuture); - return; - } - refData.put(sideInfo.getEqualFieldList().get(i), equalObj); - } - - String rowKeyStr = ((HbaseAsyncSideInfo)sideInfo).getRowKeyBuilder().getRowKey(refData); - - //get from cache - if (openCache()) { - CacheObj val = getFromCache(rowKeyStr); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.SingleLine == val.getType()) { - try { - Row row = fillData(inputCopy.row(), val); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - for (Object one : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), one); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); - } - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } - return; - } - } + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + rowKeyMode.asyncGetData(tableName, buildCacheKey(inputParams), input, resultFuture, sideInfo.getSideCache()); + } - rowKeyMode.asyncGetData(tableName, rowKeyStr, inputCopy, resultFuture, sideInfo.getSideCache()); + @Override + public String buildCacheKey(Map inputParams) { + return ((HbaseAsyncSideInfo)sideInfo).getRowKeyBuilder().getRowKey(inputParams); } @Override public Row fillData(Row input, Object sideInput){ - List sideInputList = (List) sideInput; Row row = new Row(sideInfo.getOutFieldInfoList().size()); for(Map.Entry entry : sideInfo.getInFieldIndex().entrySet()){ diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e534a85f9..e3200c293 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -128,26 +128,18 @@ private void connKuDu() throws KuduException { scannerBuilder.setProjectedColumnNames(projectColumns); } - @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - //scannerBuilder 设置为null重新加载过滤条件 + //scannerBuilder 设置为null重新加载过滤条件,然后connkudu重新赋值 + //todo:代码需要优化 scannerBuilder = null; connKuDu(); - JsonArray inputParams = new JsonArray(); Schema schema = table.getSchema(); // @wenbaoup fix bug - for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Object equalObj = inputCopy.row().getField(sideInfo.getEqualValIndex().get(i)); - if (equalObj == null) { - dealMissKey(inputCopy, resultFuture); - return; - } - //增加过滤条件 - scannerBuilder.addPredicate(KuduPredicate.newInListPredicate(schema.getColumn(sideInfo.getEqualFieldList().get(i)), Collections.singletonList(equalObj))); - inputParams.add(equalObj); - } + inputParams.entrySet().forEach(e ->{ + scannerBuilder.addPredicate(KuduPredicate.newInListPredicate(schema.getColumn(e.getKey()), Collections.singletonList(e.getValue()))); + }); // 填充谓词信息 List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -161,46 +153,23 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce }).count(); } - - String key = buildCacheKey(inputParams); - - if (openCache()) { - //判断数据是否已经加载到缓存中 - CacheObj val = getFromCache(key); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.SingleLine == val.getType()) { - try { - Row row = fillData(inputCopy.row(), val); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), jsonArray); - rowList.add(new CRow(row, inputCopy.change())); - } - resultFuture.complete(rowList); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); - } - return; - } - } List> cacheContent = Lists.newArrayList(); AsyncKuduScanner asyncKuduScanner = scannerBuilder.build(); List rowList = Lists.newArrayList(); Deferred data = asyncKuduScanner.nextRows(); //从之前的同步修改为调用异步的Callback - data.addCallbackDeferring(new GetListRowCB(inputCopy, cacheContent, rowList, asyncKuduScanner, resultFuture, key)); + data.addCallbackDeferring(new GetListRowCB(inputCopy, cacheContent, rowList, asyncKuduScanner, resultFuture, buildCacheKey(inputParams))); + } + + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()) + .append("_"); + } + + return sb.toString(); } @@ -230,16 +199,6 @@ public Row fillData(Row input, Object sideInput) { return row; } - public String buildCacheKey(JsonArray jsonArray) { - StringBuilder sb = new StringBuilder(); - for (Object ele : jsonArray.getList()) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - @Override public void close() throws Exception { super.close(); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index d85a9990d..99cc3b2db 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -102,19 +102,11 @@ public void connMongoDb() throws Exception { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); BasicDBObject basicDbObject = new BasicDBObject(); - for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(inputCopy, resultFuture); - return; - } - basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); - } try { + basicDbObject.putAll(inputParams); // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); @@ -127,27 +119,8 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDbObject.values()); - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(inputCopy, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - List rowList = Lists.newArrayList(); - for (Object jsonArray : (List) val.getContent()) { - Row row = fillData(inputCopy.row(), jsonArray); - rowList.add(new CRow(row, inputCopy.change())); - } - resultFuture.complete(rowList); - } else { - throw new RuntimeException("not support cache obj type " + val.getType()); - } - return; - } - } + String key = buildCacheKey(inputParams); + AtomicInteger atomicInteger = new AtomicInteger(0); MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); @@ -178,6 +151,17 @@ public void onResult(final Void result, final Throwable t) { dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); } + @Override + public String buildCacheKey(Map inputParams) { + StringBuilder sb = new StringBuilder(); + for (Object ele : inputParams.values()) { + sb.append(ele.toString()) + .append("_"); + } + + return sb.toString(); + } + @Override public Row fillData(Row input, Object line) { Document doc = (Document) line; @@ -216,14 +200,4 @@ public void close() throws Exception { } } - public String buildCacheKey(Collection collection) { - StringBuilder sb = new StringBuilder(); - for (Object ele : collection) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - } From 118d0fd5feef1f97a26be1e271f55f9b6ee9c59a Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 18 Mar 2020 15:57:04 +0800 Subject: [PATCH 034/378] =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/operator/SideAsyncOperator.java | 4 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 164 +++++++++--------- .../sql/side/redis/RedisAsyncReqRow.java | 55 ++---- 3 files changed, 90 insertions(+), 133 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 3c2010907..b23de8102 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -66,10 +66,10 @@ public static DataStream getSideJoinDataStream(DataStream inputStream, String si //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ - return AsyncDataStream.orderedWait(inputStream, asyncDbReq, sideTableInfo.getAsyncTimeout(), TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) + return AsyncDataStream.orderedWait(inputStream, asyncDbReq, -1, TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) .setParallelism(sideTableInfo.getParallelism()); }else { - return AsyncDataStream.unorderedWait(inputStream, asyncDbReq, sideTableInfo.getAsyncTimeout(), TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) + return AsyncDataStream.unorderedWait(inputStream, asyncDbReq, -1, TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) .setParallelism(sideTableInfo.getParallelism()); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index f78c30c30..5e5de7e38 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -25,10 +25,12 @@ import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; +import io.vertx.core.json.Json; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -37,8 +39,12 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; /** * Date: 2018/11/26 @@ -69,94 +75,45 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient SQLClient rdbSqlClient; + private Logger logger = LoggerFactory.getLogger(getClass()); + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } - @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow copyCrow = new CRow(input.row(), input.change()); - JsonArray inputParams = new JsonArray(); - for (Integer conValIndex : sideInfo.getEqualValIndex()) { - Object equalObj = copyCrow.row().getField(conValIndex); - if (equalObj == null) { - dealMissKey(copyCrow, resultFuture); - return; - } - inputParams.add(equalObj); - } - String key = buildCacheKey(inputParams); - if (openCache()) { - CacheObj val = getFromCache(key); - if (val != null) { - if (ECacheContentType.MissVal == val.getType()) { - dealMissKey(copyCrow, resultFuture); - return; - } else if (ECacheContentType.MultiLine == val.getType()) { - try { - List rowList = getRows(copyCrow, null, (List) val.getContent()); - resultFuture.complete(rowList); - } catch (Exception e) { - dealFillDataError(resultFuture, e, copyCrow); - } - } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); - } - return; - } - } + @Override + protected void preInvoke(CRow input, ResultFuture resultFuture){ - rdbSqlClient.getConnection(conn -> { - if (conn.failed()) { - //Treatment failures - resultFuture.completeExceptionally(conn.cause()); - return; - } + } - final SQLConnection connection = conn.result(); - String sqlCondition = sideInfo.getSqlCondition(); - connection.queryWithParams(sqlCondition, inputParams, rs -> { - if (rs.failed()) { - LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.completeExceptionally(rs.cause()); - return; - } - List cacheContent = Lists.newArrayList(); - List results = rs.result().getResults(); - if (results.size() > 0) { - try { - List rowList = getRows(copyCrow, cacheContent, results); - dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); - resultFuture.complete(rowList); - } catch (Exception e){ - dealFillDataError(resultFuture, e, copyCrow); + @Override + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + AtomicBoolean connectFinish = new AtomicBoolean(false); + AtomicInteger counter = new AtomicInteger(0); + while(!connectFinish.get()){ + rdbSqlClient.getConnection(conn -> { + if(conn.failed()){ + if(counter.get() % 1000 == 0){ + logger.error("getConnection error", conn.cause()); } - } else { - dealMissKey(copyCrow, resultFuture); - dealCacheData(key, CacheMissVal.getMissKeyObj()); - } - - // and close the connection - connection.close(done -> { - if (done.failed()) { - throw new RuntimeException(done.cause()); + if(counter.incrementAndGet() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(conn.cause()); + connectFinish.set(true); + return; } - }); + } + connectFinish.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); }); - }); + } } - protected List getRows(CRow inputRow, List cacheContent, List results) { - List rowList = Lists.newArrayList(); - for (JsonArray line : results) { - Row row = fillData(inputRow.row(), line); - if (null != cacheContent && openCache()) { - cacheContent.add(line); - } - rowList.add(new CRow(row, inputRow.change())); - } - return rowList; + @Override + public String buildCacheKey(Map inputParam) { + return StringUtils.join(inputParam.values(),"_"); } @Override @@ -195,18 +152,53 @@ public void close() throws Exception { } - public String buildCacheKey(JsonArray jsonArray) { - StringBuilder sb = new StringBuilder(); - for (Object ele : jsonArray.getList()) { - sb.append(ele.toString()) - .append("_"); - } - - return sb.toString(); - } - public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } + private void handleQuery(SQLConnection connection,Map inputParams, CRow input, ResultFuture resultFuture){ + String key = buildCacheKey(inputParams); + JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); + connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { + if (rs.failed()) { + LOG.error("Cannot retrieve the data from the database", rs.cause()); + resultFuture.completeExceptionally(rs.cause()); + return; + } + + List cacheContent = Lists.newArrayList(); + + int resultSize = rs.result().getResults().size(); + if (resultSize > 0) { + List rowList = Lists.newArrayList(); + + for (JsonArray line : rs.result().getResults()) { + Row row = fillData(input.row(), line); + if (openCache()) { + cacheContent.add(line); + } + rowList.add(new CRow(row, input.change())); + } + + if (openCache()) { + putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); + } + + resultFuture.complete(rowList); + } else { + dealMissKey(input, resultFuture); + if (openCache()) { + putCache(key, CacheMissVal.getMissKeyObj()); + } + } + + // and close the connection + connection.close(done -> { + if (done.failed()) { + throw new RuntimeException(done.cause()); + } + }); + }); + } + } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d2a2aed56..b6b6c5a9d 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -126,49 +126,13 @@ public Row fillData(Row input, Object sideInput) { } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(),input.change()); - List keyData = Lists.newLinkedList(); - for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { - Integer conValIndex = sideInfo.getEqualValIndex().get(i); - Object equalObj = inputCopy.row().getField(conValIndex); - if(equalObj == null){ - dealMissKey(inputCopy, resultFuture); - return; - } - String value = equalObj.toString(); - keyData.add(sideInfo.getEqualFieldList().get(i)); - keyData.add(value); - } - - String key = buildCacheKey(keyData); - - if(openCache()){ - CacheObj val = getFromCache(key); - if(val != null){ - if(ECacheContentType.MissVal == val.getType()){ - dealMissKey(inputCopy, resultFuture); - return; - }else if(ECacheContentType.MultiLine == val.getType()){ - try { - Row row = fillData(inputCopy.row(), val.getContent()); - resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); - } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); - } - }else{ - RuntimeException exception = new RuntimeException("not support cache obj type " + val.getType()); - resultFuture.completeExceptionally(exception); - } - return; - } - } - + public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + String key = buildCacheKey(inputParams); Map keyValue = Maps.newHashMap(); List value = async.keys(key + ":*").get(); String[] values = value.toArray(new String[value.size()]); if (values.length == 0) { - dealMissKey(inputCopy, resultFuture); + dealMissKey(input, resultFuture); } else { RedisFuture>> future = ((RedisStringAsyncCommands) async).mget(values); future.thenAccept(new Consumer>>() { @@ -181,14 +145,14 @@ public void accept(List> keyValues) { keyValue.put(splitKeys[3], keyValues.get(i).getValue()); } try { - Row row = fillData(inputCopy.row(), keyValue); + Row row = fillData(input.row(), keyValue); dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, keyValue)); - resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, inputCopy); + dealFillDataError(resultFuture, e, input); } } else { - dealMissKey(inputCopy, resultFuture); + dealMissKey(input, resultFuture); dealCacheData(key, CacheMissVal.getMissKeyObj()); } } @@ -196,8 +160,9 @@ public void accept(List> keyValues) { } } - private String buildCacheKey(List keyData) { - String kv = String.join(":", keyData); + @Override + public String buildCacheKey(Map inputParams) { + String kv = StringUtils.join(inputParams.values(), ":"); String tableName = redisSideTableInfo.getTableName(); StringBuilder preKey = new StringBuilder(); preKey.append(tableName).append(":").append(kv); From 1cb21db06ca2dce2bdc9cb0e53a984647151a692 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 18 Mar 2020 16:35:32 +0800 Subject: [PATCH 035/378] kafka avro and csv ,retract sink --- README.md | 24 +- docs/kafkaSink.md | 223 ++++++++++++++++++ docs/kafkaSource.md | 184 +-------------- .../kafka/AbstractKafkaProducerFactory.java | 5 +- .../AvroCRowSerializationSchema.java | 44 +++- .../CsvCRowSerializationSchema.java | 39 ++- .../sql/sink/kafka/table/KafkaSinkParser.java | 3 + .../sink/kafka/table/KafkaSinkTableInfo.java | 28 +++ 8 files changed, 324 insertions(+), 226 deletions(-) create mode 100644 docs/kafkaSink.md diff --git a/README.md b/README.md index ccf94f1ff..c84d77d5a 100644 --- a/README.md +++ b/README.md @@ -7,34 +7,11 @@ > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus - ## 新特性: - * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 - * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 - * 3.异步维表支持非等值连接，比如：<>,<,>。 - * 4.增加kafka数组解析 - * 5.增加kafka1.0以上版本的支持 - * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 - * 7.支持插件的依赖方式,参考pluginLoadMode参数 - * 8.支持cep处理 - * 9.支持udaf - * 10.支持谓词下移 - * 11.支持状态的ttl - - ## BUG修复： - * 1.修复不能解析sql中orderby,union语法。 - * 2.修复yarnPer模式提交失败的异常。 - * 3.一些bug的修复 - # 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver -# 后续开发计划 - * 维表快照 - * kafka avro格式 - * topN - ## 1 快速起步 ### 1.1 运行模式 @@ -205,6 +182,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * [impala 结果表插件](docs/impalaSink.md) * [db2 结果表插件](docs/db2Sink.md) * [sqlserver 结果表插件](docs/sqlserverSink.md) +* [kafka 结果表插件](docs/kafkaSink.md) ### 2.3 维表插件 * [hbase 维表插件](docs/hbaseSide.md) diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md new file mode 100644 index 000000000..3c6eb1dc6 --- /dev/null +++ b/docs/kafkaSink.md @@ -0,0 +1,223 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + function(colNameX) AS aliasName, + WATERMARK FOR colName AS withOffset( colName , delayTime ) + )WITH( + type ='kafka11', + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + groupId='test', + parallelism ='parllNum', + ); +``` + +## 2.支持的版本 + kafka09,kafka10,kafka11及以上版本 + **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| +|groupId | 需要读取的 groupId 名称|否|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|parallelism | 并行度设置|否|1| +|partitionKeys | 用来分区的字段|否|| +|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| +|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| +|fieldDelimiter | csv数据分隔符|否| \ | + + +**kafka相关参数可以自定义，使用kafka.开头即可。** +``` +kafka.consumer.id +kafka.socket.timeout.ms +kafka.fetch.message.max.bytes +kafka.num.consumer.fetchers +kafka.auto.commit.enable +kafka.auto.commit.interval.ms +kafka.queued.max.message.chunks +kafka.rebalance.max.retries +kafka.fetch.min.bytes +kafka.fetch.wait.max.ms +kafka.rebalance.backoff.ms +kafka.refresh.leader.backoff.ms +kafka.consumer.timeout.ms +kafka.exclude.internal.topics +kafka.partition.assignment.strategy +kafka.client.id +kafka.zookeeper.session.timeout.ms +kafka.zookeeper.connection.timeout.ms +kafka.zookeeper.sync.time.ms +kafka.offsets.storage +kafka.offsets.channel.backoff.ms +kafka.offsets.channel.socket.timeout.ms +kafka.offsets.commit.max.retries +kafka.dual.commit.enabled +kafka.partition.assignment.strategy +kafka.socket.receive.buffer.bytes +kafka.fetch.min.bytes +``` + +## 5.样例： + +### json格式： +``` +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + partitionKeys = 'channel,pv', + updateMode='upsert' + ); + +upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} +append模式下发的数据格式：{"channel":"zs","pv":"330"} + +``` + +### avro格式： + +如果updateMode='upsert',schemaInfo需要包含retract属性信息。 + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + --type='console' + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='1', + updateMode='upsert', + sinkdatatype = 'avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} + ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, + {"name":"retract","type":"boolean"}]}' + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` +### csv格式： + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='2', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + updateMode='upsert', + sinkdatatype = 'csv', + fieldDelimiter='*' + + + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 76096aba6..9b488863e 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -1,14 +1,12 @@ ## 1.格式： ``` -数据现在支持json格式{"xx":"bb","cc":"dd"} - CREATE TABLE tableName( colName colType, ... function(colNameX) AS aliasName, WATERMARK FOR colName AS withOffset( colName , delayTime ) )WITH( - type ='kafka09', + type ='kafka11', bootstrapServers ='ip:port,ip:port...', zookeeperQuorum ='ip:port,ip:port/zkparent', offsetReset ='latest', @@ -89,7 +87,7 @@ CREATE TABLE MyTable( xctime bigint, CHARACTER_LENGTH(channel) AS timeLeng )WITH( - type ='kafka09', + type ='kafka11', bootstrapServers ='172.16.8.198:9092', zookeeperQuorum ='172.16.8.198:2181/kafka', offsetReset ='latest', @@ -208,181 +206,3 @@ CREATE TABLE MyTable( lengthcheckpolicy = 'PAD' ); ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** - -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** -``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' - ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** -``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index ebd313b29..9dcaf222b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -75,15 +75,16 @@ private SerializationSchema createSerializationSchema(KafkaSinkTableInfo k if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); + serSchemaBuilder.setUpdateMode(kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = serSchemaBuilder.build(); } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(),kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java index 34fa22c99..692e208b5 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -36,8 +36,6 @@ import org.apache.avro.util.Utf8; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.formats.avro.AvroRowDeserializationSchema; -import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; @@ -55,17 +53,16 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.TimeZone; +import java.util.stream.Collectors; /** * Serialization schema that serializes CROW into Avro bytes. * *

Serializes objects that are represented in (nested) Flink rows. It support types that * are compatible with Flink's Table & SQL API. - * - *

Note: Changes in this class need to be kept in sync with the corresponding runtime - * class {@link AvroRowDeserializationSchema} and schema converter {@link AvroSchemaConverter}. - * + ** * @author maqi */ public class AvroCRowSerializationSchema implements SerializationSchema { @@ -107,14 +104,14 @@ public class AvroCRowSerializationSchema implements SerializationSchema { private String updateMode; - private final String retractKey = "retract"; + private String retractKey = "retract"; /** * Creates an Avro serialization schema for the given specific record class. * * @param recordClazz Avro record class used to serialize Flink's row to Avro's record */ - public AvroCRowSerializationSchema(Class recordClazz) { + public AvroCRowSerializationSchema(Class recordClazz, String updateMode) { Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); this.recordClazz = recordClazz; this.schema = SpecificData.get().getSchema(recordClazz); @@ -122,6 +119,7 @@ public AvroCRowSerializationSchema(Class recordClazz) this.datumWriter = new SpecificDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; } /** @@ -152,10 +150,10 @@ public byte[] serialize(CRow crow) { // convert to record final GenericRecord record = convertRowToAvroRecord(schema, row); + + dealRetractField(change, record); + arrayOutputStream.reset(); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - record.put(retractKey, change); - } datumWriter.write(record, encoder); encoder.flush(); return arrayOutputStream.toByteArray(); @@ -164,6 +162,18 @@ public byte[] serialize(CRow crow) { } } + protected void dealRetractField(boolean change, GenericRecord record) { + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .findFirst() + .ifPresent(field -> { + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + record.put(retractKey, convertFlinkType(field.schema(), change)); + } + }); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -184,7 +194,12 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { - final List fields = schema.getFields(); + + final List fields = schema.getFields() + .stream() + .filter(field -> !StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .collect(Collectors.toList()); + final int length = fields.size(); final GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < length; i++) { @@ -328,6 +343,8 @@ private long convertFromTimestamp(Schema schema, Timestamp date) { private void writeObject(ObjectOutputStream outputStream) throws IOException { outputStream.writeObject(recordClazz); outputStream.writeObject(schemaString); // support for null + outputStream.writeObject(retractKey); + outputStream.writeObject(updateMode); } @SuppressWarnings("unchecked") @@ -339,6 +356,9 @@ private void readObject(ObjectInputStream inputStream) throws ClassNotFoundExcep } else { schema = new Schema.Parser().parse(schemaString); } + retractKey = (String) inputStream.readObject(); + updateMode = (String) inputStream.readObject(); + datumWriter = new SpecificDatumWriter<>(schema); arrayOutputStream = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java index 903395f9d..4e57b6f2a 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -47,7 +47,10 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; +import java.util.Iterator; import java.util.Objects; +import java.util.stream.IntStream; +import java.util.stream.Stream; /** * Serialization schema that serializes an object of Flink types into a CSV bytes. @@ -72,25 +75,27 @@ public final class CsvCRowSerializationSchema implements SerializationSchema kafkaParam = new HashMap(); @@ -119,9 +129,27 @@ public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); + + if (StringUtils.equalsIgnoreCase(getSinkDataType(), FormatType.AVRO.name())) { + avroParamCheck(); + } + return false; } + public void avroParamCheck() { + Preconditions.checkNotNull(schemaString, "avro type schemaInfo is required"); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + Schema schema = new Schema.Parser().parse(schemaString); + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), RETRACT_FIELD_KEY)) + .findFirst() + .orElseThrow(() -> + new NullPointerException(String.valueOf("arvo upsert mode the retract attribute must be contained in schemaInfo field "))); + } + } + public String getEnableKeyPartition() { return enableKeyPartition; } From d124305ffe07b66f8eee430b3c0ac1072281dc14 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 19 Mar 2020 18:37:48 +0800 Subject: [PATCH 036/378] add jacoco --- pom.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pom.xml b/pom.xml index 5493645e9..de963240c 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,20 @@ + + org.jacoco + jacoco-maven-plugin + 0.7.8 + + + + prepare-agent + report + + + + + pl.project13.maven git-commit-id-plugin From 25316e3d705a5626b0a01a72e9e57e1693ace60d Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 19 Mar 2020 20:07:41 +0800 Subject: [PATCH 037/378] add kafka charsetname --- .../dtnest/DtNestRowDeserializationSchema.java | 9 +++++++-- .../source/kafka/AbstractKafkaConsumerFactory.java | 4 ++-- .../sql/source/kafka/table/KafkaSourceParser.java | 1 + .../sql/source/kafka/table/KafkaSourceTableInfo.java | 12 ++++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 78b579305..b6bf377ed 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -58,17 +58,22 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; private List fieldExtraInfos; + private String charsetName; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, + List fieldExtraInfos, + String charsetName) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; this.fieldExtraInfos = fieldExtraInfos; + this.charsetName = charsetName; } @Override public Row deserialize(byte[] message) throws IOException { - JsonNode root = objectMapper.readTree(message); + String decoderStr = new String(message, charsetName); + JsonNode root = objectMapper.readTree(decoderStr); this.parseTree(root, null); Row row = new Row(fieldNames.length); diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java index 2cae221cf..49680ea63 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java @@ -56,8 +56,8 @@ protected DeserializationMetricWrapper createDeserializationMetricWrapper(KafkaS private DeserializationSchema createDeserializationSchema(KafkaSourceTableInfo kafkaSourceTableInfo, TypeInformation typeInformation) { DeserializationSchema deserializationSchema = null; if (FormatType.DT_NEST.name().equalsIgnoreCase(kafkaSourceTableInfo.getSourceDataType())) { - deserializationSchema = new DtNestRowDeserializationSchema(typeInformation, kafkaSourceTableInfo.getPhysicalFields(), kafkaSourceTableInfo.getFieldExtraInfoList()); - + deserializationSchema = new DtNestRowDeserializationSchema(typeInformation, kafkaSourceTableInfo.getPhysicalFields(), + kafkaSourceTableInfo.getFieldExtraInfoList(),kafkaSourceTableInfo.getCharsetName()); } else if (FormatType.JSON.name().equalsIgnoreCase(kafkaSourceTableInfo.getSourceDataType())) { if (StringUtils.isNotBlank(kafkaSourceTableInfo.getSchemaString())) { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 005bfb4c2..6d84a057a 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -52,6 +52,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Fri, 20 Mar 2020 09:37:01 +0800 Subject: [PATCH 038/378] socket decoder --- .../serversocket/CustomerSocketTextStreamFunction.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index e46e0a76f..bdcea4672 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -64,12 +64,16 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { private transient Socket currentSocket; + private String CHARSET_NAME = "UTF-8"; + ServersocketSourceTableInfo tableInfo; + + public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; - this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); + this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos, CHARSET_NAME); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); } From 615ea8363bf60c1ee57c586a0fc64720a6fc8535 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 20 Mar 2020 15:55:18 +0800 Subject: [PATCH 039/378] =?UTF-8?q?=E5=BC=82=E5=B8=B8=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/BaseAsyncReqRow.java | 10 ++++++-- .../sql/side/rdb/async/RdbAsyncReqRow.java | 23 +++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index ae987e142..221bf2c85 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -30,6 +30,7 @@ import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.MapUtils; +import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -62,6 +63,7 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; + private RuntimeContext runtimeContext; private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; @@ -72,7 +74,11 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } - + @Override + public void setRuntimeContext(RuntimeContext runtimeContext) { + super.setRuntimeContext(runtimeContext); + this.runtimeContext = runtimeContext; + } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); @@ -221,7 +227,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result public abstract String buildCacheKey(Map inputParams); private ProcessingTimeService getProcessingTimeService(){ - return ((StreamingRuntimeContext)this.getRuntimeContext()).getProcessingTimeService(); + return ((StreamingRuntimeContext)this.runtimeContext).getProcessingTimeService(); } protected ScheduledFuture registerTimer(CRow input, ResultFuture resultFuture){ diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 5e5de7e38..6b4298f24 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -89,25 +89,28 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { - AtomicBoolean connectFinish = new AtomicBoolean(false); - AtomicInteger counter = new AtomicInteger(0); - while(!connectFinish.get()){ + AtomicInteger failCounter = new AtomicInteger(0); + while(true){ + AtomicBoolean connectFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { + connectFinish.set(true); if(conn.failed()){ - if(counter.get() % 1000 == 0){ + if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } - if(counter.incrementAndGet() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(conn.cause()); - connectFinish.set(true); - return; - } + conn.result().close(); } - connectFinish.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); }); + while(!connectFinish.get()){ + Thread.sleep(50); + } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(new RuntimeException("connection fail")); + return; + } } } From baabfd45555949eb8ead829da31e6d1c492dfbbd Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 20 Mar 2020 16:41:31 +0800 Subject: [PATCH 040/378] extract abstractKafkaSource --- .../dtstack/flink/sql/util/DtStringUtil.java | 8 +- .../sql/source/kafka/AbstractKafkaSource.java | 117 ++++++++++++ .../sql/source/kafka/enums/EKafkaOffset.java | 31 ++++ .../source/kafka/table/KafkaSourceParser.java | 29 +-- .../kafka/table/KafkaSourceTableInfo.java | 175 ++++++++---------- .../flink/sql/source/kafka/KafkaSource.java | 74 +------- .../flink/sql/source/kafka/KafkaSource.java | 72 +------ .../flink/sql/source/kafka/KafkaSource.java | 87 ++------- .../flink/sql/source/kafka/KafkaSource.java | 90 ++------- 9 files changed, 292 insertions(+), 391 deletions(-) create mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java create mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..10a6b4f63 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -219,14 +219,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return preStr + "?" + sb.toString(); } - public static boolean isJosn(String str){ + public static boolean isJson(String str) { boolean flag = false; - if(StringUtils.isNotBlank(str)){ + if (StringUtils.isNotBlank(str)) { try { - objectMapper.readValue(str,Map.class); + objectMapper.readValue(str, Map.class); flag = true; } catch (Throwable e) { - flag=false; + flag = false; } } return flag; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java new file mode 100644 index 000000000..852a381e2 --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.source.kafka; + +import com.dtstack.flink.sql.source.IStreamSourceGener; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; +import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.Table; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSource implements IStreamSourceGener

{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected Properties getKafkaProperties(KafkaSourceTableInfo kafkaSourceTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSourceTableInfo.getBootstrapServers()); + + if (DtStringUtil.isJson(kafkaSourceTableInfo.getOffsetReset())) { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, EKafkaOffset.NONE.name().toLowerCase()); + } else { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceTableInfo.getOffsetReset()); + } + + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { + props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceTableInfo.getGroupId()); + } + + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + return props; + } + + protected String generateOperatorName(String tabName, String topicName) { + return SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", tabName); + } + + protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { + Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + + return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + } + + protected void setStartPosition(String offset, String topicName, FlinkKafkaConsumerBase kafkaSrc) { + if (StringUtils.equalsIgnoreCase(offset, EKafkaOffset.EARLIEST.name())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJson(offset)) { + Map specificStartupOffsets = buildOffsetMap(offset, topicName); + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } else { + kafkaSrc.setStartFromLatest(); + } + } + + /** + * kafka offset,eg.. {"0":12312,"1":12321,"2":12312} + * @param offsetJson + * @param topicName + * @return + */ + protected Map buildOffsetMap(String offsetJson, String topicName) { + try { + Properties properties = PluginUtil.jsonStrToObject(offsetJson, Properties.class); + Map offsetMap = PluginUtil.objectToMap(properties); + Map specificStartupOffsets = offsetMap + .entrySet() + .stream() + .collect(Collectors.toMap( + (Map.Entry entry) -> new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), + (Map.Entry entry) -> Long.valueOf(entry.getValue().toString())) + ); + + return specificStartupOffsets; + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + offsetJson); + } + } + +} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java new file mode 100644 index 000000000..476ccc8b7 --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.source.kafka.enums; + +/** + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi + */ +public enum EKafkaOffset { + + LATEST, + EARLIEST, + NONE +} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 8b0b9753c..867f48d6a 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,11 +19,13 @@ package com.dtstack.flink.sql.source.kafka.table; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; import com.dtstack.flink.sql.table.AbstractSourceParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -37,28 +39,27 @@ public class KafkaSourceParser extends AbstractSourceParser { public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - kafkaSourceTableInfo.setName(tableName); - kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setName(tableName); + kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); - if (bootstrapServer == null || "".equals(bootstrapServer.trim())) { - throw new Exception("BootstrapServers can not be empty!"); - } else { - kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); - } + kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); - kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); + kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase(), EKafkaOffset.LATEST.name().toLowerCase()))); kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - for (String key : props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafkaSourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } + + Map kafkaParams = props.keySet().stream() + .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) + .collect(Collectors.toMap( + key -> key.substring(6), key -> props.get(key).toString()) + ); + + kafkaSourceTableInfo.addKafkaParam(kafkaParams); kafkaSourceTableInfo.check(); + return kafkaSourceTableInfo; } } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 364cbff49..e1c014a69 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -17,7 +17,6 @@ */ - package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.format.FormatType; @@ -37,134 +36,118 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - - public static final String TOPIC_KEY = "topic"; - - public static final String TYPE_KEY = "type"; + public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - public static final String GROUPID_KEY = "groupId"; + public static final String TOPIC_KEY = "topic"; - public static final String OFFSETRESET_KEY = "offsetReset"; + public static final String TYPE_KEY = "type"; - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String GROUPID_KEY = "groupId"; - private String bootstrapServers; + public static final String OFFSETRESET_KEY = "offsetReset"; - private String topic; + public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - private String groupId; + private String bootstrapServers; - //latest, earliest - private String offsetReset = "latest"; + private String topic; - private String offset; + private String groupId; - private Boolean topicIsPattern = false; + private String offsetReset; - private String sourceDataType = FormatType.DT_NEST.name(); + private Boolean topicIsPattern = false; - private String schemaString; + private String sourceDataType = FormatType.DT_NEST.name(); - private String fieldDelimiter; + private String schemaString; - public String getBootstrapServers() { - return bootstrapServers; - } + private String fieldDelimiter; - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } + public Map kafkaParam = new HashMap<>(); - public String getTopic() { - return topic; - } - public void setTopic(String topic) { - this.topic = topic; - } + public String getBootstrapServers() { + return bootstrapServers; + } - public String getGroupId() { - return groupId; - } + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } - public void setGroupId(String groupId) { - this.groupId = groupId; - } + public String getTopic() { + return topic; + } - public String getOffsetReset() { - return offsetReset; - } + public void setTopic(String topic) { + this.topic = topic; + } - public void setOffsetReset(String offsetReset) { - if(offsetReset == null){ - return; - } - this.offsetReset = offsetReset; - } + public String getGroupId() { + return groupId; + } - public String getOffset() { - return offset; - } + public void setGroupId(String groupId) { + this.groupId = groupId; + } - public void setOffset(String offset) { - if (offsetReset == null) { - return; - } - this.offset = offset; - } + public String getOffsetReset() { + return offsetReset; + } - public Boolean getTopicIsPattern() { - return topicIsPattern; - } + public void setOffsetReset(String offsetReset) { + this.offsetReset = offsetReset; + } - public void setTopicIsPattern(Boolean topicIsPattern) { - this.topicIsPattern = topicIsPattern; - } + public Boolean getTopicIsPattern() { + return topicIsPattern; + } - public Map kafkaParam = new HashMap<>(); + public void setTopicIsPattern(Boolean topicIsPattern) { + this.topicIsPattern = topicIsPattern; + } - public void addKafkaParam(String key, String value) { - kafkaParam.put(key, value); - } + public void addKafkaParam(Map kafkaParam) { + kafkaParam.putAll(kafkaParam); + } - public String getKafkaParam(String key) { - return kafkaParam.get(key); - } + public String getKafkaParam(String key) { + return kafkaParam.get(key); + } - public Set getKafkaParamKeys() { - return kafkaParam.keySet(); - } + public Set getKafkaParamKeys() { + return kafkaParam.keySet(); + } - public String getSourceDataType() { - return sourceDataType; - } + public String getSourceDataType() { + return sourceDataType; + } - public void setSourceDataType(String sourceDataType) { - this.sourceDataType = sourceDataType; - } + public void setSourceDataType(String sourceDataType) { + this.sourceDataType = sourceDataType; + } - public String getSchemaString() { - return schemaString; - } + public String getSchemaString() { + return schemaString; + } - public void setSchemaString(String schemaString) { - this.schemaString = schemaString; - } + public void setSchemaString(String schemaString) { + this.schemaString = schemaString; + } - public String getFieldDelimiter() { - return fieldDelimiter; - } + public String getFieldDelimiter() { + return fieldDelimiter; + } - public void setFieldDelimiter(String fieldDelimiter) { - this.fieldDelimiter = fieldDelimiter; - } + public void setFieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 7e2d77c27..394ea86ee 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -18,24 +18,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -43,75 +36,24 @@ * @create: 2019-11-05 10:55 * @description: **/ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") @Override public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index ffb466a4c..9f8917761 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -17,17 +17,13 @@ */ - package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; @@ -36,7 +32,6 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -47,74 +42,23 @@ * @author xuchao */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") +public class KafkaSource extends AbstractKafkaSource { @Override public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafkaSourceTableInfo.) + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for(int i = 0; i< kafkaSourceTableInfo.getFieldClasses().length; i++){ - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())){ - kafkaSrc.setStartFromEarliest(); - }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for(Map.Entry entry:offsetMap.entrySet()){ - specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - }else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } \ No newline at end of file diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index ff4aed89d..23989ab7e 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,74 +40,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - - FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } - - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } \ No newline at end of file diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 11be1898a..f58d59d05 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,16 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; - -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,76 +39,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); - - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafka09SourceTableInfo.) - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } +public class KafkaSource extends AbstractKafkaSource { - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } From 0b22f9c7d5c776ad7356b731c95a8704e4dd5530 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 20 Mar 2020 17:20:46 +0800 Subject: [PATCH 041/378] add avro params --- docs/kafkaSource.md | 224 ++---------------- .../source/kafka/table/KafkaSourceParser.java | 5 + .../kafka/table/KafkaSourceTableInfo.java | 8 +- 3 files changed, 38 insertions(+), 199 deletions(-) diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 76096aba6..00c252c99 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -1,6 +1,5 @@ ## 1.格式： ``` -数据现在支持json格式{"xx":"bb","cc":"dd"} CREATE TABLE tableName( colName colType, @@ -15,9 +14,8 @@ CREATE TABLE tableName( topic ='topicName', groupId='test', parallelism ='parllNum', - --timezone='America/Los_Angeles', timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 + sourcedatatype ='dt_nest' #可不设置 ); ``` @@ -47,7 +45,9 @@ CREATE TABLE tableName( |topicIsPattern | topic是否是正则表达式格式(true|false) |否| false |offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| +|sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| +|schemaInfo | avro类型使用的schema信息|否|| +|fieldDelimiter |csv类型使用的数据分隔符|否| | | |timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` @@ -169,24 +169,10 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` -# 二、csv格式数据源 -根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 -## 1.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置 |否|1| -|sourcedatatype | 数据类型|是 |csv| -|fielddelimiter | 字段分隔符|是 || -|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| -**kafka相关参数可以自定义，使用kafka.开头即可。** -## 2.样例： +## 7.csv格式数据源 + + ``` CREATE TABLE MyTable( name varchar, @@ -203,186 +189,28 @@ CREATE TABLE MyTable( --topic ='mqTest.*', --topicIsPattern='true' parallelism ='1', - sourcedatatype ='csv', - fielddelimiter ='\|', - lengthcheckpolicy = 'PAD' + sourceDatatype ='csv' ); ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** +## 8.avro格式数据源 -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** ``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false', + kafka.group.id='mqTest', + sourceDataType ='avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"},{"name":"pv","type":"string"}]}' ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** + ``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` + diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 867f48d6a..96ccd1783 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.source.kafka.table; +import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; import com.dtstack.flink.sql.table.AbstractSourceParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -51,6 +52,10 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParams = props.keySet().stream() .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) .collect(Collectors.toMap( diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index e1c014a69..c27eee376 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -48,6 +48,12 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String SCHEMA_STRING_KEY = "schemaInfo"; + + public static final String CSV_FIELD_DELIMITER_KEY = "fieldDelimiter"; + + public static final String SOURCE_DATA_TYPE_KEY = "sourceDataType"; + private String bootstrapServers; private String topic; @@ -58,7 +64,7 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { private Boolean topicIsPattern = false; - private String sourceDataType = FormatType.DT_NEST.name(); + private String sourceDataType; private String schemaString; From a78738963037df956cacf51bdca24f58b5f3165e Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 15:51:38 +0800 Subject: [PATCH 042/378] =?UTF-8?q?=E8=A1=A5=E5=85=85=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E8=BE=93=E5=87=BAleft=20join?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 221bf2c85..8713b541a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -145,11 +145,15 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } timeOutNum ++; - if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ - resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); - } else { + if(sideInfo.getJoinType() == JoinType.LEFT){ resultFuture.complete(null); + return; + } + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Integer.MAX_VALUE)){ + resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); + return; } + resultFuture.complete(null); } protected void preInvoke(CRow input, ResultFuture resultFuture){ From 23b410c04682342f37be1b785e10f14640e97b1e Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 16:03:52 +0800 Subject: [PATCH 043/378] =?UTF-8?q?=E8=BE=93=E5=87=BA=E5=A0=86=E6=A0=88?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 6b4298f24..10b9d93db 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -30,6 +30,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; @@ -45,6 +46,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * Date: 2018/11/26 @@ -90,6 +92,7 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicInteger failCounter = new AtomicInteger(0); + AtomicReference connErrMsg = new AtomicReference<>(); while(true){ AtomicBoolean connectFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { @@ -98,17 +101,20 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } + connErrMsg.set(conn.cause()); conn.result().close(); } ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); }); + while(!connectFinish.get()){ Thread.sleep(50); } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(new RuntimeException("connection fail")); + resultFuture.completeExceptionally(connErrMsg.get()); return; } } From bfce8c88b479658f9ab02d7b303fb178b6089815 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 16:26:47 +0800 Subject: [PATCH 044/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=80=80=E5=87=BA?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 10b9d93db..c48635bf3 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -93,30 +93,32 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicInteger failCounter = new AtomicInteger(0); AtomicReference connErrMsg = new AtomicReference<>(); - while(true){ + AtomicBoolean finishFlag = new AtomicBoolean(false); + while(!finishFlag.get()){ AtomicBoolean connectFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { - connectFinish.set(true); if(conn.failed()){ + connectFinish.set(true); if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(connErrMsg.get()); + finishFlag.set(true); + } connErrMsg.set(conn.cause()); conn.result().close(); + return; } ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); }); while(!connectFinish.get()){ Thread.sleep(50); } - - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(connErrMsg.get()); - return; - } } } From 267511a2659e2fbc7ba162293af75f6333f7f964 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 16:27:36 +0800 Subject: [PATCH 045/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=80=80=E5=87=BA?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index c48635bf3..827d94dad 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -92,7 +92,6 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicInteger failCounter = new AtomicInteger(0); - AtomicReference connErrMsg = new AtomicReference<>(); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ AtomicBoolean connectFinish = new AtomicBoolean(false); @@ -103,10 +102,9 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul logger.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(connErrMsg.get()); + resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } - connErrMsg.set(conn.cause()); conn.result().close(); return; } From 4e8a00e4ccf12b865f163e2d87a923be53cb9fce Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 24 Mar 2020 17:39:43 +0800 Subject: [PATCH 046/378] =?UTF-8?q?=E9=98=BB=E5=A1=9E=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E9=87=8D=E6=96=B0=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 827d94dad..011920afe 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -43,6 +43,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -77,6 +78,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient SQLClient rdbSqlClient; + private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); + private Logger logger = LoggerFactory.getLogger(getClass()); public RdbAsyncReqRow(BaseSideInfo sideInfo) { @@ -91,13 +94,32 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + + while (!CONN_STATUS.get()){//network is unhealth + Thread.sleep(100); + } + rdbSqlClient.getConnection(conn -> { + if(conn.failed()){ + CONN_STATUS.set(false); + connectWithRetry(inputParams, input, resultFuture, rdbSqlClient); + return; + } + CONN_STATUS.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); + }); + + } + + private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { AtomicInteger failCounter = new AtomicInteger(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ - AtomicBoolean connectFinish = new AtomicBoolean(false); + AtomicBoolean connFinish = new AtomicBoolean(false); rdbSqlClient.getConnection(conn -> { + connFinish.set(true); if(conn.failed()){ - connectFinish.set(true); if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } @@ -108,16 +130,22 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul conn.result().close(); return; } + CONN_STATUS.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); finishFlag.set(true); }); + while(!connFinish.get()){ + try { + Thread.sleep(50); + } catch (InterruptedException e){ + logger.error("", e); + } - while(!connectFinish.get()){ - Thread.sleep(50); } } + } @Override From e0af118f6beca7965960f06341a9d9f672a181e8 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Tue, 24 Mar 2020 21:46:17 +0800 Subject: [PATCH 047/378] =?UTF-8?q?oceanbase-sink=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 6 -- oceanbase/oceanbase-sink/pom.xml | 90 +++++++++++++++++++ .../sql/sink/ocean/OceanbaseDialect.java | 59 ++++++++++++ .../flink/sql/sink/ocean/OceanbaseSink.java | 42 +++++++++ .../sink/ocean/table/OceanbaseSinkParser.java | 25 ++++++ oceanbase/pom.xml | 39 ++++++++ pom.xml | 1 + 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 oceanbase/oceanbase-sink/pom.xml create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java create mode 100644 oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java create mode 100644 oceanbase/pom.xml diff --git a/core/pom.xml b/core/pom.xml index eb470a3c0..f57c1ea47 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -116,12 +116,6 @@ flink-statebackend-rocksdb_2.11 ${flink.version} - - - junit - junit - 4.12 - com.aiweiergou tools-logger diff --git a/oceanbase/oceanbase-sink/pom.xml b/oceanbase/oceanbase-sink/pom.xml new file mode 100644 index 000000000..02514f0d1 --- /dev/null +++ b/oceanbase/oceanbase-sink/pom.xml @@ -0,0 +1,90 @@ + + + + sql.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.sink.oceanbase + jar + oceanbase-sink + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.sink.rdb + ${sql.sink.rdb.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java new file mode 100644 index 000000000..b10f4a3b3 --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java @@ -0,0 +1,59 @@ +package com.dtstack.flink.sql.sink.ocean; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author : tiezhu + * @date : 2020/3/24 + */ +public class OceanbaseDialect implements JDBCDialect { + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.mysql.jdbc.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + @Override + public Optional getUpsertStatement(String schema, + String tableName, + String[] fieldNames, + String[] uniqueKeyFields, + boolean allReplace) { + return allReplace ? + buildReplaceIntoStatement(tableName, fieldNames) : + buildDuplicateUpsertStatement(tableName, fieldNames); + } + + private Optional buildDuplicateUpsertStatement(String tableName, String[] fieldsName) { + String updateClause = Arrays.stream(fieldsName).map(f -> quoteIdentifier(f) + + "IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(",")); + return Optional.of(getInsertIntoStatement("", tableName, fieldsName, null) + + " ON DUPLICATE KEY UPDATE " + updateClause + ); + } + + private Optional buildReplaceIntoStatement(String tableName, String[] fieldsNames) { + String columns = Arrays.stream(fieldsNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(",")); + String placeholders = Arrays.stream(fieldsNames) + .map(f -> "?") + .collect(Collectors.joining(",")); + return Optional.of("REPLACE INTO " + quoteIdentifier(tableName) + + "(" + columns + ") VALUES (" + placeholders + ")"); + } +} diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java new file mode 100644 index 000000000..c08cf9bb9 --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java @@ -0,0 +1,42 @@ +package com.dtstack.flink.sql.sink.ocean; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; + +/** + * @author : tiezhu + * @date : 2020/3/24 + */ +public class OceanbaseSink extends AbstractRdbSink implements IStreamSinkGener { + + private static final String OCEANBESE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseSink() { + super(new OceanbaseDialect()); + } + + @Override + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions oceanbaseOptions = JDBCOptions.builder() + .setDbUrl(dbUrl) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(oceanbaseOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode) + .build(); + } +} diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java new file mode 100644 index 000000000..f9f3fb42a --- /dev/null +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java @@ -0,0 +1,25 @@ +package com.dtstack.flink.sql.sink.ocean.table; + +import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author : tiezhu + * @date : 2020/3/24 + */ +public class OceanbaseSinkParser extends RdbSinkParser { + + private static final String CURRENT_TYPE = "oceanbase"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + + AbstractTableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + + oceanbaseTableInfo.setType(CURRENT_TYPE); + + return oceanbaseTableInfo; + } +} diff --git a/oceanbase/pom.xml b/oceanbase/pom.xml new file mode 100644 index 000000000..231665282 --- /dev/null +++ b/oceanbase/pom.xml @@ -0,0 +1,39 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.oceanbase + pom + + oceanbase-sink + + + + 5.1.40 + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.core + ${sql.core.version} + provided + + + + mysql + mysql-connector-java + ${mysql.connector.version} + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index de963240c..87c8e97e6 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ impala db2 polardb + oceanbase From 201dd7480e045d628cc3cf5ca5eb7779c0c5b1a7 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 24 Mar 2020 21:50:57 +0800 Subject: [PATCH 048/378] fix --- .../java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java | 1 - 1 file changed, 1 deletion(-) diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java index c08cf9bb9..8aede349b 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java @@ -3,7 +3,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** From ad272dee5efa5225fc0d8e1183144396b70f1315 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Mar 2020 15:07:30 +0800 Subject: [PATCH 049/378] fix sonarqube scan --- .../side/cassandra/CassandraAsyncReqRow.java | 3 +-- .../cassandra/CassandraAsyncSideInfo.java | 8 +++++-- .../sql/sink/console/ConsoleOutputFormat.java | 2 +- .../sink/console/table/TablePrintUtil.java | 2 +- .../flink/sql/exec/ExecuteProcessHelper.java | 9 ++------ .../DtNestRowDeserializationSchema.java | 3 ++- .../flink/sql/option/OptionParser.java | 21 +++++++------------ .../dtstack/flink/sql/side/BaseAllReqRow.java | 6 +++++- .../flink/sql/side/JoinNodeDealer.java | 11 +++++----- .../dtstack/flink/sql/side/SideSQLParser.java | 18 +--------------- .../dtstack/flink/sql/side/SideSqlExec.java | 10 ++------- .../com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../Elasticsearch6AsyncReqRow.java | 7 ++++++- .../flink/sql/side/hbase/HbaseAllReqRow.java | 14 ++++++++++--- .../PreRowKeyModeDealerDealer.java | 3 +-- .../rowkeydealer/RowKeyEqualModeDealer.java | 3 +-- .../sink/kafka/CustomerFlinkPartition.java | 8 +++---- .../sql/launcher/ClusterClientFactory.java | 4 ++-- .../perjob/PerJobClusterClientBuilder.java | 3 ++- .../sql/launcher/perjob/PerJobSubmitter.java | 1 - .../sql/side/rdb/all/RdbAllSideInfo.java | 5 ++++- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 6 +++++- .../rdb/format/JDBCUpsertOutputFormat.java | 1 - .../sink/rdb/writer/AbstractUpsertWriter.java | 3 --- 24 files changed, 71 insertions(+), 82 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index e1ed6d8e1..7eee3cb5f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -216,7 +216,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce connCassandraDB(cassandraSideTableInfo); String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - System.out.println("sqlCondition:" + sqlCondition); + LOG.info("sqlCondition:{}" + sqlCondition); ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -265,7 +265,6 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); - System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 82055b94c..4bee5648b 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -30,6 +30,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -42,6 +44,8 @@ public class CassandraAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4403313049809013362L; + private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName()); + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); @@ -63,9 +67,9 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf } sqlCondition = "select ${selectField} from ${tableName}"; - sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - System.out.println("---------side_exe_sql-----\n" + sqlCondition); + + LOG.info("---------side_exe_sql-----\n{}" + sqlCondition); } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index e99c76d03..fc5a768c4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -56,7 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { - System.out.println("received oriainal data:" + tuple2); + LOG.info("received oriainal data:{}" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 6ddc60386..12bb2a5d4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -95,7 +95,7 @@ public static TablePrintUtil build(List data) { try { value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - e.printStackTrace(); + LOG.error("", e); } item[j] = value == null ? "null" : value; } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c02c3b142..dbe38044e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -95,11 +95,8 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { LOG.info("------------program params-------------------------"); - System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); - Arrays.stream(args).forEach(System.out::println); LOG.info("-------------------------------------------"); - System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); Options options = optionParser.getOptions(); @@ -228,12 +225,10 @@ private static void sqlTranslation(String localSqlPluginPath, //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { - System.out.println("----------exec sql without dimension join-----------"); - System.out.println("----------real sql exec is--------------------------"); - System.out.println(result.getExecSql()); + LOG.info("----------exec sql without dimension join-----------"); + LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { - System.out.println(); LOG.info("exec sql: " + result.getExecSql()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 4c0d68eb2..5f1b1c6f3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -28,8 +28,9 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import java.io.IOException; diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index e49adfd93..0fd057c5a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -31,7 +31,10 @@ import java.io.File; import java.io.FileInputStream; import java.net.URLEncoder; +import java.util.stream.Stream; + import org.apache.commons.codec.Charsets; +import org.apache.flink.util.FileUtils; /** @@ -92,19 +95,16 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.objectToMap(properties); + Map mapConf = PluginUtil.objectToMap(properties); List args = Lists.newArrayList(); - for(Map.Entry one : mapConf.entrySet()){ + for (Map.Entry one : mapConf.entrySet()) { String key = one.getKey(); Object value = one.getValue(); - if(value == null){ + if (value == null) { continue; - }else if(OPTION_SQL.equalsIgnoreCase(key)){ + } else if (OPTION_SQL.equalsIgnoreCase(key)) { File file = new File(value.toString()); - FileInputStream in = new FileInputStream(file); - byte[] filecontent = new byte[(int) file.length()]; - in.read(filecontent); - String content = new String(filecontent, Charsets.UTF_8.name()); + String content = FileUtils.readFile(file, "UTF-8"); value = URLEncoder.encode(content, Charsets.UTF_8.name()); } args.add("-" + key); @@ -112,9 +112,4 @@ public List getProgramExeArgList() throws Exception { } return args; } - - public static void main(String[] args) throws Exception { - OptionParser optionParser = new OptionParser(args); - System.out.println(optionParser.getOptions()); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 674239449..0b7fe15b3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -27,6 +27,8 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.util.concurrent.ScheduledExecutorService; @@ -42,6 +44,8 @@ public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; @@ -59,7 +63,7 @@ public BaseAllReqRow(BaseSideInfo sideInfo){ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); - System.out.println("----- all cacheRef init end-----"); + LOG.info("----- all cacheRef init end-----"); //start reload cache thread AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index a1c3cecb1..411de9380 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.util.TableUtils; +import com.esotericsoftware.minlog.Log; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -264,11 +265,10 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, SqlBasicCall SqlBasicCall sqlBasicCall = buildAsSqlNode(tableAlias, sqlNode); queueInfo.offer(sqlBasicCall); - System.out.println("-------build temporary query-----------"); - System.out.println(tmpSelectSql); - System.out.println("---------------------------------------"); - }catch (Exception e){ - e.printStackTrace(); + Log.info("-------build temporary query-----------\n{}", tmpSelectSql); + Log.info("---------------------------------------"); + } catch (Exception e) { + Log.error("", e); throw new RuntimeException(e); } } @@ -389,7 +389,6 @@ private void extractSelectField(SqlNode selectNode, } }else if(selectNode.getKind() == CASE){ - System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 9be1551aa..d21effb0c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -22,46 +22,32 @@ import com.dtstack.flink.sql.config.CalciteConfig; import com.dtstack.flink.sql.util.TableUtils; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Queues; import com.google.common.collect.Sets; -import org.apache.calcite.sql.JoinType; -import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlBinaryOperator; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWith; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.api.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.*; +import static org.apache.calcite.sql.SqlKind.IDENTIFIER; /** * Parsing sql, obtain execution information dimension table @@ -76,8 +62,6 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { - System.out.println("----------exec original Sql----------"); - System.out.println(exeSql); LOG.info("----------exec original Sql----------"); LOG.info(exeSql); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index e828bec03..1a7b05416 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -143,11 +143,9 @@ public void exec(String sql, Map sideTableMap, St } if(pollSqlNode.getKind() == INSERT){ - System.out.println("----------real exec sql-----------" ); - System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + pollSqlNode.toString()); + LOG.info("----------real exec sql-----------\n{}", pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ @@ -178,8 +176,7 @@ public void exec(String sql, Map sideTableMap, St } }else if (pollObj instanceof JoinInfo){ - System.out.println("----------exec join info----------"); - System.out.println(pollObj.toString()); + LOG.info("----------exec join info----------\n{}", pollObj.toString()); preIsSideJoin = true; joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); } @@ -408,8 +405,6 @@ private void replaceFieldName(SqlNode sqlNode, FieldReplaceInfo replaceInfo) { } } }else{ - //TODO - System.out.println(sqlNode); throw new RuntimeException("---not deal type:" + sqlNode); } @@ -644,7 +639,6 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, FieldReplaceInfo repl return selectNode; }else if(selectNode.getKind() == CASE){ - System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 8708f86d7..966b82e3e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -219,7 +219,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); + return BigDecimal.valueOf(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index e8164edb2..1f4bd1bf1 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -26,7 +32,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 6f7720472..ed2931fff 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -182,9 +182,17 @@ private void loadData(Map> tmpCache) throws SQLExcep LOG.error("", e); } finally { try { - conn.close(); - table.close(); - resultScanner.close(); + if (null != conn && !conn.isClosed()) { + conn.close(); + } + + if (null != table) { + table.close(); + } + + if (null != resultScanner) { + resultScanner.close(); + } } catch (IOException e) { LOG.error("", e); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index 3fad216b2..dcd50131d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -110,8 +110,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, for (String key : colNames) { Object val = sideMap.get(key); if (val == null) { - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 01f43b246..41208c7f3 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -86,8 +86,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF for(String key : colNames){ Object val = sideMap.get(key); if(val == null){ - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index e212d1f57..90dfe996b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,13 +11,13 @@ public class CustomerFlinkPartition extends FlinkKafkaPartitioner { public CustomerFlinkPartition() { } - + @Override public void open(int parallelInstanceId, int parallelInstances) { Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); this.parallelInstanceId = parallelInstanceId; } - + @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ @@ -25,11 +25,11 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int } return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } - + @Override public boolean equals(Object o) { return this == o || o instanceof CustomerFlinkPartition; } - + @Override public int hashCode() { return CustomerFlinkPartition.class.hashCode(); } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 7f4ded520..8f0090db5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,11 +21,11 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; +import com.esotericsoftware.minlog.Log; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -109,7 +109,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - System.out.println("applicationId=" + applicationId.toString()); + Log.info("applicationId={}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 3de1cdbc5..0e5089bc9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; +import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -79,7 +80,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - System.out.println("----init yarn success ----"); + Log.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 29cc4890a..ca2d5bdd7 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -73,7 +73,6 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); - System.out.println(tips); LOG.info(tips); return applicationId; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 08dc56149..143017133 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -31,6 +31,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -47,6 +49,7 @@ public class RdbAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -5858335638589472159L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAllSideInfo.class.getSimpleName()); public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { @@ -57,7 +60,7 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); @@ -74,7 +78,7 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), equalFieldList, sqlJoinCompareOperate, sideTableInfo.getPredicateInfoes()); - System.out.println("----------dimension sql query-----------\n" + sqlCondition); + LOG.info("----------dimension sql query-----------\n{}", sqlCondition); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index b01595bb9..e8b6dc8a4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -114,7 +114,6 @@ public void open(int taskNumber, int numTasks) throws IOException { if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); LOG.info("execute insert sql： {}", insertSql); - System.out.println("execute insert sql :" + insertSql); jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); } else { jdbcWriter = AbstractUpsertWriter.create( diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 9ca0fd754..2c070b680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -70,11 +70,9 @@ public static AbstractUpsertWriter create( String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); LOG.info("deleteSQL is :{}", deleteSql); - System.out.println("deleteSQL is :" + deleteSql); Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); - System.out.println("execute UpsertStatement: " + upsertSql.orElse("use UsingInsertUpdateStatement")); return upsertSql.map((Function) sql -> new UpsertWriterUsingUpsertStatement( @@ -171,7 +169,6 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { - System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); From 7bcb8ac449c61656cb3eb2542c5d7a23a2a1fce6 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 25 Mar 2020 15:25:09 +0800 Subject: [PATCH 050/378] remove unuse class --- .../flink/sql/side/redis/RedisAllReqRow.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index f484c12b3..5d09c8f3d 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -36,21 +36,27 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; -import java.util.*; + +import java.util.Calendar; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; /** * @author yanxi */ From 2353b7fdcd52f1f96ccb9ae06480954222285bfb Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 26 Mar 2020 10:41:47 +0800 Subject: [PATCH 051/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0sonar=20=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E8=B4=A8=E9=87=8F=E6=A3=80=E6=9F=A5=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 10 ++++++++++ ci/sonar_notify.sh | 14 ++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 ci/sonar_notify.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..029098c87 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,10 @@ +build: + stage: test + script: + - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - sh ci/sonar_notify.sh + only: + - 1.8_dev + tags: + - dt-insight-engine \ No newline at end of file diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh new file mode 100644 index 000000000..41f8a3c0e --- /dev/null +++ b/ci/sonar_notify.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq + sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) + curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ + -H "Content-Type: application/json" \ + -d "{ + \"msgtype\": \"markdown\", + \"markdown\": { + \"title\":\"sonar代码质量\", + \"text\": \"## sonar代码质量报告: \n +> [sonar地址](http://172.16.100.198:9000/dashboard?id=dt-insight-engine/flinkStreamSQL) \n +> ${sonarreport} \n\" + } + }" \ No newline at end of file From b169a349e4ab14461731c64104932005f7f52fea Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 26 Mar 2020 10:52:33 +0800 Subject: [PATCH 052/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 462eed30b..a8d69597d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -37,7 +37,7 @@ /** * - * Reason: TODO ADD REASON(可选) + * 日期工具 * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com * @author sishu.yss From 45013fab4abb379a6043a7bc02a877b966c33b8b Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 26 Mar 2020 11:01:23 +0800 Subject: [PATCH 053/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 029098c87..39e0d4746 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,6 @@ build: - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - - 1.8_dev + - v1.8.0_dev tags: - dt-insight-engine \ No newline at end of file From adcf84727685e82b83aaa41e1d8a660a3b27d667 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 26 Mar 2020 16:28:40 +0800 Subject: [PATCH 054/378] oceanbase dimension --- .../flink/sql/exec/ExecuteProcessHelper.java | 6 +- .../dtstack/flink/sql/side/SideSqlExec.java | 2 +- .../sql/watermarker/WaterMarkerAssigner.java | 6 +- .../flink/sql/launcher/LauncherMain.java | 19 ++-- .../oceanbase-side/oceanbase-all-side/pom.xml | 94 +++++++++++++++++++ .../side/oceanbase/OceanbaseAllReqRow.java | 71 ++++++++++++++ .../side/oceanbase/OceanbaseAllSideInfo.java | 40 ++++++++ .../oceanbase-async-side/pom.xml | 92 ++++++++++++++++++ .../side/oceanbase/OceanbaseAsyncReqRow.java | 73 ++++++++++++++ .../oceanbase/OceanbaseAsyncSideInfo.java | 40 ++++++++ .../oceanbase-side-core/pom.xml | 18 ++++ .../oceanbase/table/OceanbaseSideParser.java | 39 ++++++++ oceanbase/oceanbase-side/pom.xml | 34 +++++++ .../sql/sink/ocean/OceanbaseDialect.java | 17 ++++ .../flink/sql/sink/ocean/OceanbaseSink.java | 19 +++- .../sink/ocean/table/OceanbaseSinkParser.java | 17 ++++ oceanbase/pom.xml | 1 + .../side/rdb/all/AbstractRdbAllReqRow.java | 11 +-- .../sql/side/rdb/all/RdbAllSideInfo.java | 3 +- 19 files changed, 570 insertions(+), 32 deletions(-) create mode 100644 oceanbase/oceanbase-side/oceanbase-all-side/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java create mode 100644 oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java create mode 100644 oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java create mode 100644 oceanbase/oceanbase-side/oceanbase-side-core/pom.xml create mode 100644 oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java create mode 100644 oceanbase/oceanbase-side/pom.xml diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index c02c3b142..01e0f9d6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -94,11 +94,9 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { - LOG.info("------------program params-------------------------"); System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); Arrays.stream(args).forEach(System.out::println); - LOG.info("-------------------------------------------"); System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); @@ -288,9 +286,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getFieldTypes(), adaptTable.getSchema().getFieldNames()); DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) - .map((Tuple2 f0) -> { - return f0.f1; - }) + .map((Tuple2 f0) -> f0.f1) .returns(typeInfo); String fields = String.join(",", typeInfo.getFieldNames()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index e828bec03..4a90e7fb1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -121,7 +121,7 @@ public void exec(String sql, Map sideTableMap, St SideSQLParser sideSQLParser = new SideSQLParser(); sideSQLParser.setLocalTableCache(localTableCache); Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); - Object pollObj = null; + Object pollObj; //need clean boolean preIsSideJoin = false; diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index b0ca0335f..784aa0b29 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -41,11 +41,7 @@ public class WaterMarkerAssigner { public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ - if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ - return false; - } - - return true; + return !Strings.isNullOrEmpty(tableInfo.getEventTimeField()); } public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0b036c87c..0943eed79 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -48,6 +48,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -67,8 +68,7 @@ public class LauncherMain { private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); - String corePath = localSqlRootJar + SP + jarPath; - return corePath; + return localSqlRootJar + SP + jarPath; } public static void main(String[] args) throws Exception { @@ -85,14 +85,14 @@ public static void main(String[] args) throws Exception { Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); if(mode.equals(ClusterMode.local.name())) { - String[] localArgs = argList.toArray(new String[argList.size()]); + String[] localArgs = argList.toArray(new String[0]); Main.main(localArgs); return; } String pluginRoot = launcherOptions.getLocalSqlPluginPath(); File jarFile = new File(getLocalCoreJarPath(pluginRoot)); - String[] remoteArgs = argList.toArray(new String[argList.size()]); + String[] remoteArgs = argList.toArray(new String[0]); PackagedProgram program = new PackagedProgram(jarFile, Lists.newArrayList(), remoteArgs); String savePointPath = confProperties.getProperty(ConfigConstrant.SAVE_POINT_PATH_KEY); @@ -116,14 +116,14 @@ public static void main(String[] args) throws Exception { private static String[] parseJson(String[] args) { BufferedReader reader = null; - String lastStr = ""; + StringBuilder lastStr = new StringBuilder(); try{ FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8); reader = new BufferedReader(inputStreamReader); String tempString = null; while((tempString = reader.readLine()) != null){ - lastStr += tempString; + lastStr.append(tempString); } reader.close(); }catch(IOException e){ @@ -137,14 +137,13 @@ private static String[] parseJson(String[] args) { } } } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + Map map = JSON.parseObject(lastStr.toString(), new TypeReference>(){} ); List list = new LinkedList<>(); for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); } - String[] array = list.toArray(new String[list.size()]); - return array; + return list.toArray(new String[0]); } } diff --git a/oceanbase/oceanbase-side/oceanbase-all-side/pom.xml b/oceanbase/oceanbase-side/oceanbase-all-side/pom.xml new file mode 100644 index 000000000..544476129 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-all-side/pom.xml @@ -0,0 +1,94 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.all.oceanbase + oceanbase-all-side + jar + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.oceanbase.core + ${sql.side.oceanbase.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java new file mode 100644 index 000000000..30317a302 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllReqRow.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstatck.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Maps; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.List; +import java.util.Map; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAllReqRow extends AbstractRdbAllReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(OceanbaseAllReqRow.class); + + private static final String OCEAN_BASE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseAllReqRow(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) { + super(new OceanbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public Connection getConn(String dbUrl, String userName, String password) { + try { + Class.forName(OCEAN_BASE_DRIVER); + Map addParams = Maps.newHashMap(); + addParams.put("useCursorFetch", "true"); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + return DriverManager.getConnection(targetDbUrl, userName, password); + } catch (Exception e) { + LOG.error("oceanbase get connect error", e); + throw new RuntimeException(e); + } + } + + @Override + public int getFetchSize() { + return Integer.MIN_VALUE; + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java new file mode 100644 index 000000000..a983a82fd --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-all-side/src/main/java/com/dtstatck/flink/sql/side/oceanbase/OceanbaseAllSideInfo.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstatck.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAllSideInfo extends RdbAllSideInfo { + public OceanbaseAllSideInfo(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml b/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml new file mode 100644 index 000000000..dd5158ce3 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/pom.xml @@ -0,0 +1,92 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.async.oceanbase + oceanbase-async-side + jar + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.oceanbase.core + ${sql.side.oceanbase.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java new file mode 100644 index 000000000..fd76b56b9 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncReqRow.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.jdbc.JDBCClient; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; + +import java.util.List; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAsyncReqRow extends RdbAsyncReqRow { + + private static final String OCEAN_BASE_DRIVER = "com.mysql.jdbc.Driver"; + + public OceanbaseAsyncReqRow(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) { + super(new OceanbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + JsonObject oceanbaseClientConfig = new JsonObject(); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + oceanbaseClientConfig.put("url", rdbSideTableInfo.getUrl()) + .put("driver_class", OCEAN_BASE_DRIVER) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("user", rdbSideTableInfo.getUserName()) + .put("password", rdbSideTableInfo.getPassword()) + .put("provider_class", DT_PROVIDER_CLASS) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_conncetion_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSqlClient(JDBCClient.createNonShared(vertx, oceanbaseClientConfig)); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java new file mode 100644 index 000000000..c3e79a573 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-async-side/src/main/java/com/dtstack/flink/sql/side/oceanbase/OceanbaseAsyncSideInfo.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.oceanbase; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseAsyncSideInfo extends RdbAsyncSideInfo { + + public OceanbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, + List outfieldsInfoList, + AbstractSideTableInfo sideTableInfo){ + super(rowTypeInfo, joinInfo, outfieldsInfoList, sideTableInfo); + } +} diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml b/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml new file mode 100644 index 000000000..397b62063 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-side-core/pom.xml @@ -0,0 +1,18 @@ + + + + sql.side.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.oceanbase.core + oceanbase-side-core + 1.0-SNAPSHOT + jar + + \ No newline at end of file diff --git a/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java new file mode 100644 index 000000000..a5c6324a9 --- /dev/null +++ b/oceanbase/oceanbase-side/oceanbase-side-core/src/main/java/com/dtstack/flink/sql/side/oceanbase/table/OceanbaseSideParser.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.flink.sql.side.oceanbase.table; + +import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author : tiezhu + * @date : 2020/3/26 + */ +public class OceanbaseSideParser extends RdbSideParser { + + private static final String CURRENT_TYPE = "oceanbase"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oceanbaseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + oceanbaseTableInfo.setType(CURRENT_TYPE); + return oceanbaseTableInfo; + } +} diff --git a/oceanbase/oceanbase-side/pom.xml b/oceanbase/oceanbase-side/pom.xml new file mode 100644 index 000000000..2f25c3d24 --- /dev/null +++ b/oceanbase/oceanbase-side/pom.xml @@ -0,0 +1,34 @@ + + + + sql.oceanbase + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.oceanbase + 1.0-SNAPSHOT + + oceanbase-all-side + oceanbase-async-side + oceanbase-side-core + + oceanbase-side + pom + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.rdb + ${rdb.side.version} + + + + \ No newline at end of file diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java index b10f4a3b3..6a157ea23 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseDialect.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.ocean; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java index 8aede349b..537a2b3ea 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/OceanbaseSink.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.ocean; import com.dtstack.flink.sql.sink.IStreamSinkGener; @@ -11,8 +28,6 @@ */ public class OceanbaseSink extends AbstractRdbSink implements IStreamSinkGener { - private static final String OCEANBESE_DRIVER = "com.mysql.jdbc.Driver"; - public OceanbaseSink() { super(new OceanbaseDialect()); } diff --git a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java index f9f3fb42a..29f2e67da 100644 --- a/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java +++ b/oceanbase/oceanbase-sink/src/main/java/com/dtstack/flink/sql/sink/ocean/table/OceanbaseSinkParser.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.ocean.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; diff --git a/oceanbase/pom.xml b/oceanbase/pom.xml index 231665282..8b7c8bc36 100644 --- a/oceanbase/pom.xml +++ b/oceanbase/pom.xml @@ -13,6 +13,7 @@ pom oceanbase-sink + oceanbase-side diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 2e9986742..5a1158125 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -43,10 +43,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -107,7 +104,7 @@ public void flatMap(CRow value, Collector out) throws Exception { List equalValIndex = sideInfo.getEqualValIndex(); ArrayList inputParams = equalValIndex.stream() .map(value.row()::getField) - .filter(object -> null != object) + .filter(Objects::nonNull) .collect(Collectors.toCollection(ArrayList::new)); if (inputParams.size() != equalValIndex.size() && sideInfo.getJoinType() == JoinType.LEFT) { @@ -124,7 +121,7 @@ public void flatMap(CRow value, Collector out) throws Exception { out.collect(new CRow(fillData(value.row(), null), value.change())); } - cacheList.stream().forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); + cacheList.forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } @Override @@ -218,7 +215,7 @@ private void queryAndFillData(Map>> tmpCache, C } String cacheKey = sideInfo.getEqualFieldList().stream() - .map(equalField -> oneRow.get(equalField)) + .map(oneRow::get) .map(Object::toString) .collect(Collectors.joining("_")); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 08dc56149..7140849e2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -68,8 +68,7 @@ private String getSelectFromStatement(String tableName, List selectField String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); String whereClause = buildWhereClause(predicateClause); - String sql = "SELECT " + fromClause + " FROM " + tableName + whereClause; - return sql; + return "SELECT " + fromClause + " FROM " + tableName + whereClause; } private String buildWhereClause(String predicateClause) { From 2d9a65efe53461e9c772284bd79c414565891114 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 13:55:04 +0800 Subject: [PATCH 055/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E6=B7=BB=E5=8A=A0login-token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39e0d4746..242b309a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login= 11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From 0ae4bb331c6d081dd8ced9c8f77fbfa89223f98b Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 15:04:06 +0800 Subject: [PATCH 056/378] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=B1=BB=E5=90=8D=E7=A7=B0=E9=94=99=E8=AF=AFpackage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/java/com/dtstack/flinkx/AppTest.java | 58 ------------------- .../test/java/com/dtstack/flinkx/AppTest.java | 58 ------------------- .../test/java/com/dtstack/flinkx/AppTest.java | 58 ------------------- 3 files changed, 174 deletions(-) delete mode 100644 cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java delete mode 100644 mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java delete mode 100644 mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java diff --git a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} From eafed59c1ab647d636d0d4113a12211cdc98aa55 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 17:00:21 +0800 Subject: [PATCH 057/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20login=20?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 242b309a9..981a47148 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login= 11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From 59b996b53d3b164565cbdd683a343c8915c1e52c Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 27 Mar 2020 17:35:53 +0800 Subject: [PATCH 058/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E5=88=86?= =?UTF-8?q?=E6=94=AF=20=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 981a47148..2952551ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From c895ba2d0961d5ec07e632bc8f2de6fe06c3fcab Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 30 Mar 2020 09:56:58 +0800 Subject: [PATCH 059/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dkafka-source=20?= =?UTF-8?q?=E4=BD=9C=E4=B8=BA=E6=95=B0=E6=8D=AE=E6=BA=90=E9=94=99=E8=AF=AF?= =?UTF-8?q?=EF=BC=8Ctype=3D'kakfa'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source/kafka/KafkaDeserializationMetricWrapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index afa950c5b..d0d76747d 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -76,8 +76,13 @@ protected void beforeDeserialize() throws IOException { } protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { + Field consumerThreadField = null; + if(fetcher.getClass().getDeclaredField("consumerThread") != null){ + consumerThreadField = fetcher.getClass().getDeclaredField("consumerThread"); + } else { + consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + } - Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); From a8947ec9f4740494432542ec0ac8c947de006ef4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 30 Mar 2020 11:46:44 +0800 Subject: [PATCH 060/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=98=BB=E5=A1=9E?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 011920afe..da6cb3876 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -43,6 +43,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; @@ -116,9 +117,8 @@ private void connectWithRetry(Map inputParams, CRow input, Resul AtomicInteger failCounter = new AtomicInteger(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ - AtomicBoolean connFinish = new AtomicBoolean(false); + CountDownLatch latch = new CountDownLatch(1); rdbSqlClient.getConnection(conn -> { - connFinish.set(true); if(conn.failed()){ if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); @@ -127,22 +127,27 @@ private void connectWithRetry(Map inputParams, CRow input, Resul resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } + latch.countDown(); conn.result().close(); return; } - CONN_STATUS.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), inputParams, input, resultFuture); - finishFlag.set(true); - }); - while(!connFinish.get()){ try { - Thread.sleep(50); - } catch (InterruptedException e){ + CONN_STATUS.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); + } catch (Exception e) { logger.error("", e); + } finally { + latch.countDown(); } - + }); + //主线程阻塞 + try { + latch.wait(); + } catch (InterruptedException e) { + logger.error("", e); } } From b5c6b4a17282b4f20f9c941114b29ffd45807d5d Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 30 Mar 2020 12:02:30 +0800 Subject: [PATCH 061/378] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E7=9A=84=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kafka/AbstractKafkaProducerFactory.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 9dcaf222b..9958a2544 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -26,12 +26,9 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.formats.avro.AvroRowSerializationSchema; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; From ba3680f3ef4142826d7314b958689ddb5417f2cf Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 30 Mar 2020 13:40:52 +0800 Subject: [PATCH 062/378] csv sink default delimiter --- docs/kafkaSink.md | 2 +- .../com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md index 3c6eb1dc6..5d7c7b2a7 100644 --- a/docs/kafkaSink.md +++ b/docs/kafkaSink.md @@ -41,7 +41,7 @@ CREATE TABLE tableName( |partitionKeys | 用来分区的字段|否|| |updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| |sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| -|fieldDelimiter | csv数据分隔符|否| \ | +|fieldDelimiter | csv数据分隔符|否| , | **kafka相关参数可以自定义，使用kafka.开头即可。** diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 6740ea867..4ad8947a8 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -49,7 +49,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 30 Mar 2020 14:44:55 +0800 Subject: [PATCH 063/378] =?UTF-8?q?hbase=20=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=94=AF=E6=8C=81=20rowKey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/hbase/AbstractReplaceOperator.java | 58 +++++++ .../sql/sink/hbase/HbaseOutputFormat.java | 37 ++--- .../sql/sink/hbase/Md5ReplaceOperator.java | 43 +++++ .../flink/sql/sink/hbase/ReplaceInfo.java | 74 +++++++++ .../flink/sql/sink/hbase/RowKeyBuilder.java | 148 ++++++++++++++++++ .../sql/sink/hbase/enums/EReplaceOpType.java | 39 +++++ .../sql/sink/hbase/enums/EReplaceType.java | 43 +++++ 7 files changed, 419 insertions(+), 23 deletions(-) create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java new file mode 100644 index 000000000..c3e5701aa --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/AbstractReplaceOperator.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceOpType; + +import java.util.Map; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public abstract class AbstractReplaceOperator { + + private EReplaceOpType opType; + + public AbstractReplaceOperator(EReplaceOpType opType){ + this.opType = opType; + } + + public String doOperator(Map refData){ + String replaceStr = replaceStr(refData); + return doFunc(replaceStr); + } + + public String replaceStr(Map refData){ + return ""; + } + + /** + * The processing function to provide custom + * @param replaceStr + * @return + */ + abstract String doFunc(String replaceStr); +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 382e8f975..1769fbda7 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -23,6 +23,7 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -40,7 +41,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.text.SimpleDateFormat; import java.util.List; import java.util.Map; import java.util.Set; @@ -69,8 +69,6 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient Connection conn; private transient Table table; - public final SimpleDateFormat ROWKEY_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss"); - @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); @@ -178,30 +176,23 @@ private String buildRowKey(Row record) { private List getRowKeyValues(Row record) { List rowKeyValues = Lists.newArrayList(); - for (int i = 0; i < rowkey.length; ++i) { - String colName = rowkey[i]; - int rowKeyIndex = 0; - for (; rowKeyIndex < columnNames.length; ++rowKeyIndex) { - if (columnNames[rowKeyIndex].equals(colName)) { - break; - } - } - - if (rowKeyIndex != columnNames.length && record.getField(rowKeyIndex) != null) { - Object field = record.getField(rowKeyIndex); - if (field == null) { - continue; - } else if (field instanceof java.util.Date) { - java.util.Date d = (java.util.Date) field; - rowKeyValues.add(ROWKEY_DATE_FORMAT.format(d)); - } else { - rowKeyValues.add(field.toString()); - } - } + Map row = rowConvertMap(record); + for (String key : rowkey) { + RowKeyBuilder rowKeyBuilder = new RowKeyBuilder(); + rowKeyBuilder.init(key); + rowKeyValues.add(rowKeyBuilder.getRowKey(row)); } return rowKeyValues; } + private Map rowConvertMap(Row record){ + Map rowValue = Maps.newHashMap(); + for(int i = 0; i < columnNames.length; i++){ + rowValue.put(columnNames[i], record.getField(i)); + } + return rowValue; + } + @Override public void close() throws IOException { if (conn != null) { diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java new file mode 100644 index 000000000..e9474220a --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/Md5ReplaceOperator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceOpType; +import com.dtstack.flink.sql.util.MD5Utils; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public class Md5ReplaceOperator extends AbstractReplaceOperator { + + public Md5ReplaceOperator(EReplaceOpType opType) { + super(opType); + } + + @Override + String doFunc(String replaceStr) { + return MD5Utils.getMD5String(replaceStr); + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java new file mode 100644 index 000000000..c233bfe6b --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/ReplaceInfo.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceType; + +import java.io.Serializable; +import java.util.List; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * + * @author xuchao + */ + +public class ReplaceInfo implements Serializable { + + private static final long serialVersionUID = 2058635242957737717L; + + private EReplaceType type; + + private String param; + + private List subReplaceInfos; + + public ReplaceInfo(EReplaceType type){ + this.type = type; + } + + public EReplaceType getType() { + return type; + } + + public void setType(EReplaceType type) { + this.type = type; + } + + public String getParam() { + return param; + } + + public void setParam(String param) { + this.param = param; + } + + public List getSubReplaceInfos() { + return subReplaceInfos; + } + + public void setSubReplaceInfos(List subReplaceInfos) { + this.subReplaceInfos = subReplaceInfos; + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java new file mode 100644 index 000000000..89667ee28 --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase; + +import com.dtstack.flink.sql.sink.hbase.enums.EReplaceType; +import com.dtstack.flink.sql.util.MD5Utils; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; + +import javax.xml.crypto.Data; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * rowkey rule + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public class RowKeyBuilder implements Serializable{ + + private static final long serialVersionUID = 2058635242857937717L; + + private static Pattern Md5Operator = Pattern.compile("(?i)^md5\\(\\s*(.*)\\s*\\)$"); + + private List operatorChain = Lists.newArrayList(); + + public void init(String rowKeyTempl){ + operatorChain.addAll(makeFormula(rowKeyTempl)); + } + + /** + * + * @param refData + * @return + */ + public String getRowKey(Map refData){ + return buildStr(operatorChain, refData); + } + + + + private String buildStr(List fieldList, Map refData){ + if(CollectionUtils.isEmpty(fieldList)){ + return ""; + } + StringBuffer sb = new StringBuffer(""); + for(ReplaceInfo replaceInfo : fieldList){ + + if(replaceInfo.getType() == EReplaceType.CONSTANT){ + sb.append(replaceInfo.getParam()); + continue; + } + + if(replaceInfo.getType() == EReplaceType.FUNC){ + sb.append(MD5Utils.getMD5String(buildStr(replaceInfo.getSubReplaceInfos(), refData))); + continue; + } + String replaceName = replaceInfo.getParam(); + if(!refData.containsKey(replaceName)){ + throw new RuntimeException(String.format("build rowKey with field %s which value not found.", replaceName)); + } + Object value = refData.get(replaceName); + if(value instanceof Date){ + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + java.util.Date d = (java.util.Date) value; + sb.append(sdf.format(d)); + } else { + sb.append(value); + } + } + + return sb.toString(); + } + + public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ + String splitPatternStr = delimiter + "(?![^()]*+\\))(?![^{}]*+})(?![^\\[\\]]*+\\])"; + return str.split(splitPatternStr); + } + + /** + * + * @param field + * @return + */ + public ReplaceInfo getReplaceInfo(String field){ + + field = field.trim(); + if(field.length() <= 2){ + throw new RuntimeException(field + " \n" + + "Format defined exceptions"); + } + + //判断是不是常量==>''包裹的标识 + if(field.startsWith("'") && field.endsWith("'")){ + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.CONSTANT); + field = field.substring(1, field.length() - 1); + replaceInfo.setParam(field); + return replaceInfo; + } + + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.PARAM); + replaceInfo.setParam(field); + return replaceInfo; + } + + private List makeFormula(String formula){ + if(formula == null || formula.length() <= 0){ + Lists.newArrayList(); + } + List result = Lists.newArrayList(); + for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ + Matcher matcher = Md5Operator.matcher(meta.trim()); + if(matcher.find()){ + ReplaceInfo replaceInfo = new ReplaceInfo(EReplaceType.FUNC); + replaceInfo.setSubReplaceInfos(makeFormula(matcher.group(1))); + result.add(replaceInfo); + } else { + result.add(getReplaceInfo(meta)); + } + } + return result; + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java new file mode 100644 index 000000000..8621f352d --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceOpType.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase.enums; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * @author xuchao + */ + +public enum EReplaceOpType { + /** + * 没有func + */ + NO_FUNC, + /** + * md5 func + */ + MD5_FUNC; +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java new file mode 100644 index 000000000..ad6bbcc4e --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/enums/EReplaceType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flink.sql.sink.hbase.enums; + +/** + * Reason: + * Date: 2018/8/23 + * Company: www.dtstack.com + * + * @author xuchao + */ +public enum EReplaceType { + /** + * 参数 + */ + PARAM, + /** + * 函数 + */ + FUNC, + /** + * 常量 + */ + CONSTANT; +} From 6754700c3708b46af6cdf5dfbd8c15277665c497 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 31 Mar 2020 09:43:46 +0800 Subject: [PATCH 064/378] =?UTF-8?q?fix=20=E7=B1=BB=E5=9E=8B=E5=88=A4?= =?UTF-8?q?=E6=96=ADbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kafka/KafkaDeserializationMetricWrapper.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index d0d76747d..bf0cee7dd 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -76,13 +76,7 @@ protected void beforeDeserialize() throws IOException { } protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - Field consumerThreadField = null; - if(fetcher.getClass().getDeclaredField("consumerThread") != null){ - consumerThreadField = fetcher.getClass().getDeclaredField("consumerThread"); - } else { - consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); - } - + Field consumerThreadField = getConsumerThreadField(fetcher); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); @@ -123,4 +117,12 @@ public Long getValue() { public void setFetcher(AbstractFetcher fetcher) { this.fetcher = fetcher; } + + private Field getConsumerThreadField(AbstractFetcher fetcher) throws NoSuchFieldException { + try { + return fetcher.getClass().getDeclaredField("consumerThread"); + } catch (Exception e) { + return fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); + } + } } From 53f4f40bee5262dbcdb7b36758988806b0d013c6 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 31 Mar 2020 21:59:48 +0800 Subject: [PATCH 065/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/option/OptionParser.java | 5 +- .../sql/launcher/ClusterClientFactory.java | 2 +- .../flink/sql/launcher/LauncherMain.java | 53 +++++++++++-------- .../perjob/PerJobClusterClientBuilder.java | 4 +- .../sql/launcher/perjob/PerJobSubmitter.java | 36 +++++++------ 5 files changed, 56 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0fd057c5a..6e2413437 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.io.File; @@ -104,8 +105,8 @@ public List getProgramExeArgList() throws Exception { continue; } else if (OPTION_SQL.equalsIgnoreCase(key)) { File file = new File(value.toString()); - String content = FileUtils.readFile(file, "UTF-8"); - value = URLEncoder.encode(content, Charsets.UTF_8.name()); + String content = FileUtils.readFile(file, StandardCharsets.UTF_8.name()); + value = URLEncoder.encode(content, StandardCharsets.UTF_8.name()); } args.add("-" + key); args.add(value.toString()); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 8f0090db5..4c421fa54 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -96,7 +96,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); yarnClient.start(); - ApplicationId applicationId = null; + ApplicationId applicationId; String yarnSessionConf = launcherOptions.getYarnSessionConf(); yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0b036c87c..92259c57d 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -16,11 +16,11 @@ * limitations under the License. */ - package com.dtstack.flink.sql.launcher; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; +//import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; + import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; @@ -41,6 +41,8 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; @@ -48,6 +50,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -56,6 +59,7 @@ /** * Date: 2017/2/20 * Company: www.dtstack.com + * * @author xuchao */ @@ -64,15 +68,16 @@ public class LauncherMain { private static String SP = File.separator; + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); + private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); - String corePath = localSqlRootJar + SP + jarPath; - return corePath; + return localSqlRootJar + SP + jarPath; } public static void main(String[] args) throws Exception { - if (args.length == 1 && args[0].endsWith(".json")){ + if (args.length == 1 && args[0].endsWith(".json")) { args = parseJson(args); } OptionParser optionParser = new OptionParser(args); @@ -84,27 +89,31 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - if(mode.equals(ClusterMode.local.name())) { - String[] localArgs = argList.toArray(new String[argList.size()]); +// LOG.info("current mode is {}", mode); + System.out.println("current mode is " + mode); + + if (mode.equals(ClusterMode.local.name())) { + String[] localArgs = argList.toArray(new String[0]); Main.main(localArgs); return; } String pluginRoot = launcherOptions.getLocalSqlPluginPath(); File jarFile = new File(getLocalCoreJarPath(pluginRoot)); - String[] remoteArgs = argList.toArray(new String[argList.size()]); + String[] remoteArgs = argList.toArray(new String[0]); PackagedProgram program = new PackagedProgram(jarFile, Lists.newArrayList(), remoteArgs); String savePointPath = confProperties.getProperty(ConfigConstrant.SAVE_POINT_PATH_KEY); - if(StringUtils.isNotBlank(savePointPath)){ + if (StringUtils.isNotBlank(savePointPath)) { String allowNonRestoredState = confProperties.getOrDefault(ConfigConstrant.ALLOW_NON_RESTORED_STATE_KEY, "false").toString(); program.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savePointPath, BooleanUtils.toBoolean(allowNonRestoredState))); } - if(mode.equals(ClusterMode.yarnPer.name())){ + if (mode.equals(ClusterMode.yarnPer.name())) { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = StringUtils.isEmpty(flinkConfDir) ? new Configuration() : GlobalConfiguration.loadConfiguration(flinkConfDir); JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, config, 1); + LOG.info("current jobID is {}", jobGraph.getJobID()); PerJobSubmitter.submit(launcherOptions, jobGraph, config); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); @@ -116,20 +125,20 @@ public static void main(String[] args) throws Exception { private static String[] parseJson(String[] args) { BufferedReader reader = null; - String lastStr = ""; - try{ + StringBuilder lastStr = new StringBuilder(); + try { FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8); reader = new BufferedReader(inputStreamReader); - String tempString = null; - while((tempString = reader.readLine()) != null){ - lastStr += tempString; + String tempString; + while ((tempString = reader.readLine()) != null) { + lastStr.append(tempString); } reader.close(); - }catch(IOException e){ + } catch (IOException e) { e.printStackTrace(); - }finally{ - if(reader != null){ + } finally { + if (reader != null) { try { reader.close(); } catch (IOException e) { @@ -137,14 +146,14 @@ private static String[] parseJson(String[] args) { } } } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + Map map = JSON.parseObject(lastStr.toString(), new TypeReference>() { + }); List list = new LinkedList<>(); for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); } - String[] array = list.toArray(new String[list.size()]); - return array; + return list.toArray(new String[0]); } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 0e5089bc9..c40f9432d 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; -import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -80,7 +79,8 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - Log.info("----init yarn success ----"); + System.out.println("----init yarn success ----"); +// LOG.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index ca2d5bdd7..287844c87 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.net.URLDecoder; import java.util.Arrays; import java.util.List; @@ -41,6 +42,7 @@ * per job mode submitter * Date: 2018/11/17 * Company: www.dtstack.com + * * @author xuchao */ @@ -49,15 +51,15 @@ public class PerJobSubmitter { private static final Logger LOG = LoggerFactory.getLogger(PerJobSubmitter.class); public static String submit(Options launcherOptions, JobGraph jobGraph, Configuration flinkConfig) throws Exception { - if (!StringUtils.isBlank(launcherOptions.getAddjar())) { - String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); - List paths = getJarPaths(addjarPath); - paths.forEach( path -> { - jobGraph.addJar(new Path("file://" + path)); - }); - } - - String confProp = launcherOptions.getConfProp(); + if (!StringUtils.isBlank(launcherOptions.getAddjar())) { + String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); + List paths = getJarPaths(addjarPath); + paths.forEach(path -> { + jobGraph.addJar(new Path("file://" + path)); + }); + } + + String confProp = launcherOptions.getConfProp(); confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); ClusterSpecification clusterSpecification = FLinkPerJobResourceUtil.createClusterSpecification(confProperties); @@ -67,23 +69,23 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJarPath = launcherOptions.getFlinkJarPath(); AbstractYarnClusterDescriptor yarnClusterDescriptor = perJobClusterClientBuilder.createPerJobClusterDescriptor(flinkJarPath, launcherOptions, jobGraph); - ClusterClient clusterClient = yarnClusterDescriptor.deployJobCluster(clusterSpecification, jobGraph,true); + ClusterClient clusterClient = yarnClusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true); String applicationId = clusterClient.getClusterId().toString(); String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); + System.out.println(tips); LOG.info(tips); return applicationId; } - private static List getJarPaths(String addjarPath) { - if (addjarPath.length() > 2) { - addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); - } - List paths = Arrays.asList(StringUtils.split(addjarPath, ",")); - return paths; - } + private static List getJarPaths(String addjarPath) { + if (addjarPath.length() > 2) { + addjarPath = addjarPath.substring(1, addjarPath.length() - 1).replace("\"", ""); + } + return Arrays.asList(StringUtils.split(addjarPath, ",")); + } } From c2b4d8adaf4a667f02a15e6874898f308f934d69 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 14:59:25 +0800 Subject: [PATCH 066/378] kafkaSink opt --- .../sql/sink/kafka/AbstractKafkaSink.java | 148 ++++++++++++++++++ .../flink/sql/sink/kafka/KafkaSink.java | 120 ++------------ .../flink/sql/sink/kafka/KafkaSink.java | 125 ++------------- .../flink/sql/sink/kafka/KafkaSink.java | 125 ++------------- .../flink/sql/sink/kafka/KafkaSink.java | 123 ++------------- 5 files changed, 186 insertions(+), 455 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java new file mode 100644 index 000000000..7234216a7 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Optional; +import java.util.Properties; +import java.util.stream.IntStream; + +/** + * Date: 2020/4/1 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + public static final String SINK_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected String[] fieldNames; + protected TypeInformation[] fieldTypes; + + protected String[] partitionKeys; + protected String sinkOperatorName; + protected Properties properties; + protected int parallelism; + protected String topic; + protected String tableName; + + protected TableSchema schema; + protected SinkFunction kafkaProducer; + + + protected Optional> partitioner; + + protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaSinkTableInfo.getBootstrapServers()); + + for (String key : KafkaSinkTableInfo.getKafkaParamKeys()) { + props.setProperty(key, KafkaSinkTableInfo.getKafkaParam(key)); + } + return props; + } + + protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + return types; + } + + + protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] fieldTypes) { + Preconditions.checkArgument(fieldNames.length == fieldTypes.length, "fieldNames length must equals fieldTypes length !"); + + TableSchema.Builder builder = TableSchema.builder(); + IntStream.range(0, fieldTypes.length) + .forEach(i -> builder.field(fieldNames[i], fieldTypes[i])); + + return builder.build(); + } + + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(getRowTypeInfo()) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer).name(sinkOperatorName); + } + + public CRowTypeInfo getRowTypeInfo() { + return new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + } + + protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { + if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + +} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 71e938ba5..632bb720e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,25 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -46,108 +29,23 @@ * @create: 2019-11-05 11:45 * @description: **/ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer flinkKafkaProducer; - protected CRowTypeInfo typeInformation; - - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafkaSinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); - for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); + this.tableName = kafkaSinkTableInfo.getName(); + this.topic = kafkaSinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; - for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducerFactory() - .createKafkaProducer(kafkaSinkTableInfo, typeInformation, properties, partitioner, partitionKeys); - return this; - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; + this.fieldTypes = getTypeInformations(kafkaSinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafkaSinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index e6dbdf3d3..d22be3d59 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,26 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -49,111 +31,22 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected FlinkKafkaProducer09 kafkaProducer09; - protected CRowTypeInfo typeInformation; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - - protected int parallelism; - - - +public class KafkaSink extends AbstractKafkaSink{ @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka09SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); - for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); + this.tableName = kafka09SinkTableInfo.getName(); + this.topic = kafka09SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka09SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka09SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka09SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer09Factory().createKafkaProducer(kafka09SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer09) - .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ac5a11810..eea78e121 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -48,110 +33,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected int parallelism; - - protected KafkaSinkTableInfo kafka10SinkTableInfo; - - protected RichSinkFunction kafkaProducer010; - protected CRowTypeInfo typeInformation; - - /** The schema of the table. */ - private TableSchema schema; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { - this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka10SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); - - for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); - } + KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; + Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); + this.tableName = kafka10SinkTableInfo.getName(); + this.topic = kafka10SinkTableInfo.getTopic(); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka10SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i()), partitionKeys); - - return this; - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; + this.fieldTypes = getTypeInformations(kafka10SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka10SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 835941ca3..ea45280c7 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,25 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -51,110 +34,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer011 kafkaProducer011; - protected CRowTypeInfo typeInformation; - - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - private String[] partitionKeys; - - +public class KafkaSink extends AbstractKafkaSink { @Override public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka11SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); - for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); + this.tableName = kafka11SinkTableInfo.getName(); + this.topic = kafka11SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka11SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, typeInformation, properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka11SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka11SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer011Factory().createKafkaProducer(kafka11SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(typeInformation) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } From 7676cc7f89355bb7918a2e5fecc08361bc334c08 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 15:49:52 +0800 Subject: [PATCH 067/378] remove unuse --- .../flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index e69de29bb..000000000 From a7cff4d5010430db436cbc1a42a4558fd41db300 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 1 Apr 2020 17:58:34 +0800 Subject: [PATCH 068/378] rdb sink check connection --- .../rdb/format/RetractJDBCOutputFormat.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 28479f4f6..0f3f1f827 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -51,7 +51,8 @@ public class RetractJDBCOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(RetractJDBCOutputFormat.class); - + private static final int CONNECTION_CHECK_FREQUENCY = 100; + private int checkTimes; private String username; private String password; private String drivername; @@ -181,7 +182,6 @@ public void writeRecord(Tuple2 tuple2) { private void insertWrite(Row row) { - checkConnectionOpen(dbConn); try { if (batchNum == 1) { writeSingleRecord(row); @@ -315,6 +315,7 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { + checkConnectionOpen(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { @@ -330,11 +331,17 @@ private synchronized void submitExecuteBatch() { } } - private void checkConnectionOpen(Connection dbConn) { + private void checkConnectionOpen() { + LOG.info("test db connection Valid check !"); + checkTimes++; + if (checkTimes % CONNECTION_CHECK_FREQUENCY != 0) { + return; + } + LOG.warn("db connection Valid check !"); try { - if (dbConn.isClosed()) { + if (dbConn.isClosed() || !dbConn.isValid(100)) { LOG.info("db connection reconnect.."); - dbConn= establishConnection(); + dbConn = establishConnection(); upload = dbConn.prepareStatement(insertQuery); this.dbConn = dbConn; } @@ -345,6 +352,7 @@ private void checkConnectionOpen(Connection dbConn) { } catch (IOException e) { LOG.error("kerberos authentication failed..", e); } + checkTimes = 0; } /** From 1a375e8f801e8ebd61d4d1ae5abc3f59aeef1ac1 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 2 Apr 2020 15:34:39 +0800 Subject: [PATCH 069/378] use RestClusterClient --- .../flink/sql/launcher/ClusterClientFactory.java | 13 ++++++------- .../dtstack/flink/sql/launcher/LauncherMain.java | 3 +-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 4c421fa54..9748c5c84 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,6 +26,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.client.program.rest.RestClusterClientConfiguration; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -70,10 +72,7 @@ public static ClusterClient createClusterClient(Options launcherOptions) throws public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(config); - MiniCluster miniCluster = new MiniCluster(configBuilder.build()); - MiniClusterClient clusterClient = new MiniClusterClient(config, miniCluster); + RestClusterClient clusterClient = new RestClusterClient<>(config, "clusterClient"); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); config.setString(JobManagerOptions.ADDRESS, address.getAddress().getHostName()); @@ -122,7 +121,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { } catch (Exception e) { throw new RuntimeException(e); } - }else{ + } else { throw new RuntimeException("yarn mode must set param of 'yarnconf'!!!"); } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 92259c57d..7caf6cd63 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -89,8 +89,7 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); -// LOG.info("current mode is {}", mode); - System.out.println("current mode is " + mode); + LOG.info("current mode is {}", mode); if (mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[0]); From 64c2050759a21e436c7e68acd19b4f0147ffe1f4 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 2 Apr 2020 16:03:33 +0800 Subject: [PATCH 070/378] deal rdb sink error --- .../flink/sql/enums/EConnectionErrorCode.java | 62 +++++++++++++++++++ .../sql/side/EConnectionErrorCodeTest.java | 39 ++++++++++++ .../rdb/format/RetractJDBCOutputFormat.java | 56 ++++++++++------- 3 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java new file mode 100644 index 000000000..feed69785 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.enums; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; + +import java.util.Arrays; + +/** + * Date: 2020/4/2 + * Company: www.dtstack.com + * @author maqi + */ +public enum EConnectionErrorCode { + ERROR_NOT_MATCH(0, "错误信息未匹配", new String[]{}), + CONN_DB_INVALID(1, "数据库连接失效，请重新打开", new String[]{"the last packet successfully received from the server was"}), + CONN_DB_FAILED(2, "数据库连接失败，请检查用户名或密码是否正确", new String[]{"Access denied for user"}), + DB_TABLE_NOT_EXIST(3, "操作的表名不存在", new String[]{"doesn't exist"}); + + private int code; + private String description; + private String[] baseErrorInfo; + + EConnectionErrorCode(int code, String description, String[] baseErrorInfo) { + this.code = code; + this.description = description; + this.baseErrorInfo = baseErrorInfo; + } + + + public static EConnectionErrorCode resolveErrorCodeFromException(Throwable e) { + final String stackErrorMsg = ExceptionUtils.getFullStackTrace(e); + return Arrays.stream(values()) + .filter(errorCode -> matchKnowError(errorCode, stackErrorMsg)) + .findAny() + .orElse(ERROR_NOT_MATCH); + } + + public static boolean matchKnowError(EConnectionErrorCode errorCode, String errorMsg) { + return Arrays.stream(errorCode.baseErrorInfo) + .filter(baseInfo -> StringUtils.containsIgnoreCase(errorMsg, baseInfo)) + .findAny() + .isPresent(); + } +} diff --git a/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java new file mode 100644 index 000000000..a43b253df --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side; + +import com.dtstack.flink.sql.enums.EConnectionErrorCode; +import org.junit.Assert; +import org.junit.Test; + +/** + * Date: 2020/4/2 + * Company: www.dtstack.com + * @author maqi + */ +public class EConnectionErrorCodeTest { + + @Test + public void testResolveErrorCodeFromException(){ + EConnectionErrorCode errorCode = + EConnectionErrorCode.resolveErrorCodeFromException(new Exception("The last packet successfully received from the server was 179 milliseconds")); + + Assert.assertEquals(errorCode, EConnectionErrorCode.CONN_DB_INVALID); + } +} diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 0f3f1f827..22d7814fc 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; +import com.dtstack.flink.sql.enums.EConnectionErrorCode; import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.commons.lang3.StringUtils; @@ -181,7 +182,7 @@ public void writeRecord(Tuple2 tuple2) { } - private void insertWrite(Row row) { + private void insertWrite(Row row) { try { if (batchNum == 1) { writeSingleRecord(row); @@ -201,8 +202,6 @@ private void insertWrite(Row row) { } else { outDirtyRecords.inc(batchNum == 1 ? batchNum : rows.size()); } - - } } @@ -213,7 +212,7 @@ private void writeSingleRecord(Row row) { upload.executeUpdate(); dbConn.commit(); } catch (SQLException e) { - + dealSQLException(e); if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed,dirty record num:{}, current row:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); @@ -315,46 +314,59 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce private synchronized void submitExecuteBatch() { try { - checkConnectionOpen(); + regularlyCheckConnection(); this.upload.executeBatch(); dbConn.commit(); } catch (SQLException e) { + LOG.warn("submitExecuteBatch error {}", e); try { dbConn.rollback(); } catch (SQLException e1) { - LOG.error("rollback data error !", e); + dealSQLException(e1); } - rows.forEach(this::writeSingleRecord); } finally { rows.clear(); } } - private void checkConnectionOpen() { - LOG.info("test db connection Valid check !"); + private void dealSQLException(Exception e) { + EConnectionErrorCode errorCode = EConnectionErrorCode.resolveErrorCodeFromException(e); + switch (errorCode) { + case CONN_DB_INVALID: + reconnection(); + break; + case CONN_DB_FAILED: + case DB_TABLE_NOT_EXIST: + throw new RuntimeException(e); + default: + } + } + + + private void regularlyCheckConnection() throws SQLException { checkTimes++; if (checkTimes % CONNECTION_CHECK_FREQUENCY != 0) { return; } LOG.warn("db connection Valid check !"); - try { - if (dbConn.isClosed() || !dbConn.isValid(100)) { - LOG.info("db connection reconnect.."); - dbConn = establishConnection(); - upload = dbConn.prepareStatement(insertQuery); - this.dbConn = dbConn; - } - } catch (SQLException e) { - LOG.error("check connection open failed..", e); - } catch (ClassNotFoundException e) { - LOG.error("load jdbc class error when reconnect db..", e); - } catch (IOException e) { - LOG.error("kerberos authentication failed..", e); + if (dbConn.isClosed() || !dbConn.isValid(100)) { + reconnection(); } checkTimes = 0; } + public void reconnection() throws RuntimeException { + try { + LOG.info("db connection reconnect.."); + dbConn = establishConnection(); + upload = dbConn.prepareStatement(insertQuery); + this.dbConn = dbConn; + } catch (Exception e) { + throw new RuntimeException("connection open failed..", e); + } + } + /** * Executes prepared statement and closes all resources of this instance. * From 75516ad824776ef06ddbe77bd0a75cdb0e990332 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 2 Apr 2020 16:14:59 +0800 Subject: [PATCH 071/378] fix applicationId NullPointException --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 9748c5c84..d2b0eb928 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -157,7 +157,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) } - if (StringUtils.isEmpty(applicationId.toString())) { + if (applicationId == null || StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } return applicationId; From 5b49c0a72138ad7ab14e6521c83b1c6bfe4fee76 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 2 Apr 2020 20:57:12 +0800 Subject: [PATCH 072/378] fix --- .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 7caf6cd63..da2e881a6 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -112,8 +112,10 @@ public static void main(String[] args) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = StringUtils.isEmpty(flinkConfDir) ? new Configuration() : GlobalConfiguration.loadConfiguration(flinkConfDir); JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, config, 1); + LOG.info("current jobID is {}", jobGraph.getJobID()); - PerJobSubmitter.submit(launcherOptions, jobGraph, config); + + LOG.info("submit applicationId is {}", PerJobSubmitter.submit(launcherOptions, jobGraph, config)); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); clusterClient.run(program, 1); From 00a4564a484097816de282a05b040689c4b9269e Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 10:00:18 +0800 Subject: [PATCH 073/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9high-availab?= =?UTF-8?q?ility.cluster.id=E7=9A=84=E5=88=A4=E6=96=AD=EF=BC=8C=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=E5=88=99=E4=B8=BAapplicationId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/launcher/ClusterClientFactory.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index d2b0eb928..a6f6f3c32 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.StringHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.net.URLDecoder; @@ -59,6 +61,8 @@ */ public class ClusterClientFactory { + private static final Logger LOG = LoggerFactory.getLogger(ClusterClientFactory.class); + public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if (mode.equals(ClusterMode.standalone.name())) { @@ -100,6 +104,9 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { String yarnSessionConf = launcherOptions.getYarnSessionConf(); yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); + + LOG.info("current yarn config:\n{}", yarnSessionConfProperties); + Object yid = yarnSessionConfProperties.get("yid"); if (null != yid) { @@ -108,12 +115,18 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - Log.info("applicationId={}", applicationId.toString()); + LOG.info("applicationId={}", applicationId.toString()); + + if (config.getString("high-availability.cluster-id", null) == null) { + config.setString("high-availability.cluster-id", applicationId.toString()); + } if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } + LOG.info("current config detail:\n{}", config); + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); From 3c6c7977a8ce0b85c4d9db7a0a91367e9136bdd4 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 10:14:56 +0800 Subject: [PATCH 074/378] fix --- .../dtstack/flink/sql/launcher/ClusterClientFactory.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index a6f6f3c32..77681a603 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -117,14 +117,13 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { LOG.info("applicationId={}", applicationId.toString()); - if (config.getString("high-availability.cluster-id", null) == null) { - config.setString("high-availability.cluster-id", applicationId.toString()); - } - if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } + if (config.getString("high-availability.cluster-id", null) == null) { + config.setString("high-availability.cluster-id", applicationId.toString()); + } LOG.info("current config detail:\n{}", config); AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); From f544d1a011155af15fcab52c8b9f24f96e87a773 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 11:17:40 +0800 Subject: [PATCH 075/378] jedis npe check --- .../flink/sql/side/redis/RedisAllReqRow.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 5d09c8f3d..6c82d9902 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -103,6 +103,7 @@ protected void reloadCache() { loadData(newCache); } catch (SQLException e) { LOG.error("", e); + throw new RuntimeException(e); } cacheRef.set(newCache); @@ -160,21 +161,22 @@ private void loadData(Map> tmpCache) throws SQLExcep JedisCommands jedis = null; try { StringBuilder keyPattern = new StringBuilder(tableInfo.getTableName()); - for(String key : tableInfo.getPrimaryKeys()){ + for (String key : tableInfo.getPrimaryKeys()) { keyPattern.append("_").append("*"); - }; + } jedis = getJedisWithRetry(CONN_RETRY_NUM); + if (null == jedis) { + throw new RuntimeException("redis all load data error,get jedis commands error!"); + } Set keys = getRedisKeys(RedisType.parse(tableInfo.getRedisType()), jedis, keyPattern.toString()); - if(CollectionUtils.isEmpty(keys)){ + if (CollectionUtils.isEmpty(keys)) { return; } - for(String key : keys){ + for (String key : keys) { tmpCache.put(key, jedis.hgetAll(key)); } - } catch (Exception e){ - LOG.error("", e); } finally { - if (jedis != null){ + if (jedis != null) { try { ((Closeable) jedis).close(); } catch (IOException e) { From 1e5986949fdfc515cfb1ba2c656ee505007a1fcd Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 11:44:06 +0800 Subject: [PATCH 076/378] fix --- .../flink/sql/launcher/ClusterClientFactory.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 77681a603..5a67b4e40 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; import com.esotericsoftware.minlog.Log; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; @@ -63,6 +64,10 @@ public class ClusterClientFactory { private static final Logger LOG = LoggerFactory.getLogger(ClusterClientFactory.class); + private static final String HA_CLUSTER_ID = "high-availability.cluster-id"; + + private static final String HADOOP_CONF = "fs.hdfs.hadoopconf"; + public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if (mode.equals(ClusterMode.standalone.name())) { @@ -92,7 +97,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString("fs.hdfs.hadoopconf", yarnConfDir); + config.setString(HADOOP_CONF, yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -105,8 +110,6 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); - LOG.info("current yarn config:\n{}", yarnSessionConfProperties); - Object yid = yarnSessionConfProperties.get("yid"); if (null != yid) { @@ -115,16 +118,15 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - LOG.info("applicationId={}", applicationId.toString()); + LOG.info("current applicationId = {}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } - if (config.getString("high-availability.cluster-id", null) == null) { - config.setString("high-availability.cluster-id", applicationId.toString()); + if (config.getString(HA_CLUSTER_ID, null) == null) { + config.setString(HA_CLUSTER_ID, applicationId.toString()); } - LOG.info("current config detail:\n{}", config); AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); From fa77a64801cd475027907b941c26cf30579d358e Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 11:53:58 +0800 Subject: [PATCH 077/378] add log --- .../sql/launcher/ClusterClientFactory.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 5a67b4e40..e7ae98d73 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,21 +21,15 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; -import com.esotericsoftware.minlog.Log; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.MiniClusterClient; import org.apache.flink.client.program.rest.RestClusterClient; -import org.apache.flink.client.program.rest.RestClusterClientConfiguration; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; @@ -50,12 +44,7 @@ import java.net.InetSocketAddress; import java.net.URLDecoder; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Properties; -import java.util.Set; +import java.util.*; /** * @author sishu.yss @@ -128,6 +117,11 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { config.setString(HA_CLUSTER_ID, applicationId.toString()); } + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); From 417b1a7e1ae03e94d00b2067e282e2e229bf0034 Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 11:54:55 +0800 Subject: [PATCH 078/378] add log --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index e7ae98d73..79991f9a4 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -70,6 +70,11 @@ public static ClusterClient createClusterClient(Options launcherOptions) throws public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); + + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + RestClusterClient clusterClient = new RestClusterClient<>(config, "clusterClient"); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); From 40fb964f7cc3856cded3b20224756fdd6c7a535a Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Fri, 3 Apr 2020 14:23:04 +0800 Subject: [PATCH 079/378] fix --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 79991f9a4..1da0db7a3 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -126,7 +126,6 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); LOG.info("-------------------------------------------"); - AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); From 304f18061944f240b30e7b1c456ce85a9e28cc72 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 14:49:18 +0800 Subject: [PATCH 080/378] code opt --- .../side/cassandra/CassandraAllReqRow.java | 2 +- .../environment/MyLocalStreamEnvironment.java | 10 ++-- .../flink/sql/side/AbstractSideTableInfo.java | 4 +- .../dtstack/flink/sql/side/BaseAllReqRow.java | 2 + .../com/dtstack/flink/sql/util/DateUtil.java | 53 +++++++++---------- .../com/dtstack/flink/sql/util/JDBCUtils.java | 10 +--- .../Elasticsearch6AllReqRow.java | 2 +- .../sql/side/elasticsearch6/util/Es6Util.java | 7 --- .../side/elasticsearch6/util/MathUtil.java | 2 +- .../flink/sql/side/hbase/RowKeyBuilder.java | 5 +- .../sink/kafka/CustomerFlinkPartition.java | 1 + .../sql/launcher/ClusterClientFactory.java | 2 +- .../flink/sql/launcher/LauncherMain.java | 34 +++--------- .../flink/sql/side/mongo/MongoAllReqRow.java | 10 ++-- .../flink/sql/side/redis/RedisAllReqRow.java | 6 +-- 15 files changed, 60 insertions(+), 90 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 3ad8b934e..5f642d176 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -267,7 +267,7 @@ private void loadData(Map>> tmpCache) throws SQ String connInfo = "address:" + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 8ac1edd41..a8728c595 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -113,14 +113,16 @@ public JobExecutionResult execute(String jobName) throws Exception { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = new MiniCluster(configBuilder.build()); + MiniCluster exec = null; try { + exec = new MiniCluster(configBuilder.build()); exec.start(); return exec.executeJobBlocking(jobGraph); - } - finally { + } finally { transformations.clear(); - exec.closeAsync(); + if (null != exec) { + exec.closeAsync(); + } } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 7a832d0a1..9e114fe87 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -57,11 +57,11 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; - private String cacheType = "none";//None or LRU or ALL + private String cacheType = "none"; private int cacheSize = 10000; - private long cacheTimeout = 60 * 1000;// + private long cacheTimeout = 60_000L; private int asyncCapacity=100; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 3e3d0a8fb..ad9dcf1bd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -47,6 +47,8 @@ public abstract class BaseAllReqRow extends RichFlatMapFunction impl private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); + public static final long LOAD_DATA_ERROR_SLEEP_TIME = 5_000L; + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 409602baa..a3bb949a8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -75,7 +75,6 @@ public static Date stringToDate(String strDate) { return null; } try { - ; return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); } catch (DateTimeParseException ignored) { } @@ -113,13 +112,13 @@ public static long getTodayStart(long day) { if (("" + day).length() > 10) { cal.setTime(new Date(day)); } else { - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); } cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -131,7 +130,7 @@ public static long getTodayStart(long day) { */ public static long getTodayStart(long day,String scope) { if("MS".equals(scope)){ - return getTodayStart(day)*1000; + return getTodayStart(day)*1000L; }else if("S".equals(scope)){ return getTodayStart(day); }else{ @@ -151,13 +150,13 @@ public static long getNextDayStart(long day) { if (("" + day).length() > 10) { cal.setTime(new Date(day)); } else { - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); } cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - nextDay = (cal.getTimeInMillis() + daySpanMill) / 1000; + nextDay = (cal.getTimeInMillis() + daySpanMill) / 1000L; return nextDay; } @@ -169,7 +168,7 @@ public static long getNextDayStart(long day) { */ public static long getNextDayStart(long day,String scope) { if("MS".equals(scope)){ - return getNextDayStart(day)*1000; + return getNextDayStart(day)*1000L; }else if("S".equals(scope)){ return getNextDayStart(day); }else{ @@ -186,13 +185,13 @@ public static long getNextDayStart(long day,String scope) { public static long getMonthFirst(long day) { long firstDay = 0L; Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); cal.set(Calendar.DAY_OF_MONTH, 1); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -224,7 +223,7 @@ public static int getYear(long day) { public static long getWeekFirst(long day) { long firstDay = 0L; Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); cal.setFirstDayOfWeek(Calendar.MONDAY); cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); cal.set(Calendar.HOUR_OF_DAY, 0); @@ -243,7 +242,7 @@ public static long getWeekFirst(long day) { */ public static int getWeekOfYear(long day) { Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); return cal.get(Calendar.WEEK_OF_YEAR); } @@ -363,7 +362,7 @@ public static long getDateMillToFormat(String day, String inFormat) throws Parse Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); calendar.setTime(date); - return calendar.getTimeInMillis()/1000; + return calendar.getTimeInMillis()/1000L; } /** @@ -383,7 +382,7 @@ public static long getFirstDay4Month(int year, int month) { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -405,7 +404,7 @@ public static long getLastDay4Month(int year, int month) { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - lastDay = cal.getTimeInMillis() / 1000; + lastDay = cal.getTimeInMillis() / 1000L; return lastDay; } @@ -444,7 +443,7 @@ public static long getMillByOneDay() { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis() / 1000; + return cal.getTimeInMillis() / 1000L; } /** @@ -458,7 +457,7 @@ public static long getMillByYesDay() { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis() / 1000; + return cal.getTimeInMillis() / 1000L; } /** @@ -472,7 +471,7 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis() / 1000; + return cal.getTimeInMillis() / 1000L; } /** @@ -534,7 +533,7 @@ public static long getMillToDay(Calendar cal,int dateT){ cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; + return cal.getTimeInMillis()/1000L; } /** @@ -563,7 +562,7 @@ public static String getDate(long day, String format) { if (("" + day).length() > 10) { cal.setTime(new Date(day)); } else { - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); } SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); @@ -619,7 +618,7 @@ public static Date stringToDate(String day, String format) { */ public static String longToString(long day, String format) throws ParseException { if (("" + day).length() <= 10){ - day=day*1000; + day=day*1000L; } SimpleDateFormat dateFormat = new SimpleDateFormat(format); String date = dateFormat.format(day); @@ -634,7 +633,7 @@ public static String longToString(long day, String format) throws ParseException */ public static int getMinusDate(int day, int minusDay) { Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(day * 1000)); + cal.setTime(new Date(day * 1000L)); cal.set(Calendar.DATE, cal.get(Calendar.DATE) - minusDay); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); @@ -663,7 +662,7 @@ public static int getWeeksBetweenTwoDates(long startDay, long endDay) { public static int getMaxWeekOfYear(long startDay) { Calendar cal = Calendar.getInstance(); - cal.setTime(new Date(startDay * 1000)); + cal.setTime(new Date(startDay * 1000L)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); } @@ -696,7 +695,7 @@ public static long getMinuteStart(long time) { if (("" + time).length() > 10) { cal.setTime(new Date(time)); } else { - cal.setTime(new Date(time * 1000)); + cal.setTime(new Date(time * 1000L)); } cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); @@ -719,7 +718,7 @@ public static long getHourStart(long time) { cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); cal.set(Calendar.MINUTE, 0); - firstDay = cal.getTimeInMillis() / 1000; + firstDay = cal.getTimeInMillis() / 1000L; return firstDay; } @@ -757,7 +756,7 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { if (column == null) { return null; } else if(column instanceof String) { - return new java.sql.Timestamp(stringToDate((String)column).getTime()); + return null == stringToDate((String) column) ? null : new java.sql.Timestamp(stringToDate((String) column).getTime()); } else if (column instanceof Integer) { Integer rawData = (Integer) column; return new java.sql.Timestamp(rawData.longValue()); @@ -791,7 +790,7 @@ public static Timestamp getTimestampFromStr(String timeStr) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); } - return new Timestamp(stringToDate(timeStr).getTime()); + return null == stringToDate(timeStr) ? null : new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { @@ -803,7 +802,7 @@ public static java.sql.Date getDateFromStr(String dateStr) { Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); } - return new java.sql.Date(stringToDate(dateStr).getTime()); + return null == stringToDate(dateStr) ? null : new java.sql.Date(stringToDate(dateStr).getTime()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index 3d66d8a6d..faab22172 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -19,19 +19,13 @@ package com.dtstack.flink.sql.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.DriverManager; public class JDBCUtils { - - private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - - public final static String LOCK_STR = "jdbc_lock_str"; + private static final Object LOCK = new Object(); public static void forName(String clazz, ClassLoader classLoader) { - synchronized (LOCK_STR){ + synchronized (LOCK){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index b8cf2c46a..e50ed5edc 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -187,7 +187,7 @@ private void loadData(Map>> tmpCache) throws IO try { String connInfo = "url: " + tableInfo.getAddress() + "; userName: " + tableInfo.getUserName() + ", pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 3d9e4d62a..bc5637515 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -176,13 +176,6 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu case "NOT_EQUALS": return boolQueryBuilder.mustNot(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition())[0])); default: - try { - throw new Exception("elasticsearch6 does not support this operation: " + info.getOperatorKind()); - } catch (Exception e) { - - e.printStackTrace(); - LOG.error(e.getMessage()); - } return boolQueryBuilder; } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index d6af042f4..d72530a56 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -216,7 +216,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); + return BigDecimal.valueOf(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index b8530f224..4b3f35b68 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -120,9 +120,10 @@ public ReplaceInfo getReplaceInfo(String field){ } private List makeFormula(String formula){ - if(formula == null || formula.length() <= 0){ - Lists.newArrayList(); + if (formula == null || formula.length() <= 0) { + return Lists.newArrayList(); } + List result = Lists.newArrayList(); for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ Matcher matcher = Md5Operator.matcher(meta.trim()); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 90dfe996b..d8028fcac 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -20,6 +20,7 @@ public void open(int parallelInstanceId, int parallelInstances) { @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); + if(key == null){ return partitions[this.parallelInstanceId % partitions.length]; } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 8f0090db5..11b7dec34 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -158,7 +158,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) } - if (StringUtils.isEmpty(applicationId.toString())) { + if (null == applicationId) { throw new RuntimeException("No flink session found on yarn cluster."); } return applicationId; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0b036c87c..c39d8b8a9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -22,6 +22,7 @@ import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.launcher.perjob.PerJobClusterClientBuilder; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; @@ -41,6 +42,9 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; @@ -64,7 +68,6 @@ public class LauncherMain { private static String SP = File.separator; - private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); String corePath = localSqlRootJar + SP + jarPath; @@ -114,32 +117,11 @@ public static void main(String[] args) throws Exception { } - private static String[] parseJson(String[] args) { - BufferedReader reader = null; - String lastStr = ""; - try{ - FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); - reader = new BufferedReader(inputStreamReader); - String tempString = null; - while((tempString = reader.readLine()) != null){ - lastStr += tempString; - } - reader.close(); - }catch(IOException e){ - e.printStackTrace(); - }finally{ - if(reader != null){ - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + private static String[] parseJson(String[] args) throws IOException { + String lastStr = FileUtils.readFileUtf8(new File(args[0])); + Map map = JSON.parseObject(lastStr, new TypeReference>() { + }); List list = new LinkedList<>(); - for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 3373f3667..c7fa2398e 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -205,7 +205,7 @@ private void loadData(Map>> tmpCache) throws SQ try { String connInfo = "url:" + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } @@ -248,12 +248,8 @@ private void loadData(Map>> tmpCache) throws SQ } catch (Exception e) { LOG.error("", e); } finally { - try { - if (mongoClient != null) { - mongoClient.close(); - } - } catch (Exception e) { - throw new RuntimeException("[closeMongoDB]:" + e.getMessage()); + if (mongoClient != null) { + mongoClient.close(); } } } diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 6c82d9902..bdee79f07 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -239,17 +239,17 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { } private JedisCommands getJedisWithRetry(int retryNum) { - while (retryNum-- > 0){ + while (retryNum-- > 0) { try { return getJedis(tableInfo); } catch (Exception e) { - if(retryNum <= 0){ + if (retryNum <= 0) { throw new RuntimeException("getJedisWithRetry error", e); } try { String jedisInfo = "url:" + tableInfo.getUrl() + ",pwd:" + tableInfo.getPassword() + ",database:" + tableInfo.getDatabase(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + jedisInfo); - Thread.sleep(5 * 1000); + Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME); } catch (InterruptedException e1) { LOG.error("", e1); } From c27194e6829d5a77c360db353319deefc171ebc0 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 15:37:57 +0800 Subject: [PATCH 081/378] code opt --- .../format/SerializationMetricWrapper.java | 1 - .../flink/sql/option/OptionParser.java | 2 -- .../sql/parser/CreateTmpTableParser.java | 6 ++++- .../flink/sql/parser/InsertSqlParser.java | 9 ++++++- .../com/dtstack/flink/sql/side/JoinInfo.java | 1 - .../com/dtstack/flink/sql/util/DateUtil.java | 24 +++++++++++++++---- .../com/dtstack/flink/sql/util/MathUtil.java | 12 ---------- .../dtstack/flink/sql/util/ParseUtils.java | 13 +++++++--- .../sql/watermarker/WaterMarkerAssigner.java | 2 -- .../sink/elasticsearch/CustomerSinkFunc.java | 1 - .../flink/sql/side/hbase/HbaseAllReqRow.java | 17 ++++++++++--- .../CustomerKeyedSerializationSchema.java | 2 -- .../AvroCRowSerializationSchema.java | 2 +- .../CsvCRowSerializationSchema.java | 7 ++---- .../kafka/table/KafkaSourceTableInfo.java | 1 - .../flink/sql/sink/kafka/KafkaProducer09.java | 1 - .../sink/kafka/KafkaProducer09Factory.java | 1 - .../flink/sql/source/kafka/KafkaSource.java | 3 --- .../sink/kafka/KafkaProducer010Factory.java | 1 - .../sink/kafka/KafkaProducer011Factory.java | 1 - .../flink/sql/launcher/LauncherMain.java | 7 ------ .../perjob/PerJobClusterClientBuilder.java | 2 -- .../flink/sql/side/mongo/MongoAllReqRow.java | 2 -- .../sql/side/mongo/MongoAsyncReqRow.java | 7 ------ .../sql/sink/mongo/MongoOutputFormat.java | 2 -- .../sink/postgresql/PostgresqlDialect.java | 1 - .../sql/sink/rdb/JDBCTypeConvertUtils.java | 1 - .../flink/sql/side/redis/RedisAllReqRow.java | 1 - .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +++- 29 files changed, 62 insertions(+), 72 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 8802198a0..fda394af7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -25,7 +25,6 @@ import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; /** diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0fd057c5a..47416cd1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -29,9 +29,7 @@ import java.util.List; import java.util.Map; import java.io.File; -import java.io.FileInputStream; import java.net.URLEncoder; -import java.util.stream.Stream; import org.apache.commons.codec.Charsets; import org.apache.flink.util.FileUtils; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index de7141eb5..443e83816 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -22,7 +22,11 @@ import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import com.google.common.collect.Lists; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index a7c6db9eb..479aecc5c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -21,7 +21,14 @@ package com.dtstack.flink.sql.parser; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.lang3.StringUtils; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 8a8fe21f6..40b21ef03 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -25,7 +25,6 @@ import com.google.common.base.Strings; import java.io.Serializable; -import java.util.Map; /** * Join信息 diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a3bb949a8..a420c9d82 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -23,10 +23,21 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.*; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.util.*; + +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; +import java.util.SimpleTimeZone; +import java.util.TimeZone; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; @@ -756,7 +767,8 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { if (column == null) { return null; } else if(column instanceof String) { - return null == stringToDate((String) column) ? null : new java.sql.Timestamp(stringToDate((String) column).getTime()); + Date date = stringToDate((String) column); + return null == date ? null : new java.sql.Timestamp(date.getTime()); } else if (column instanceof Integer) { Integer rawData = (Integer) column; return new java.sql.Timestamp(rawData.longValue()); @@ -790,7 +802,8 @@ public static Timestamp getTimestampFromStr(String timeStr) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); } - return null == stringToDate(timeStr) ? null : new Timestamp(stringToDate(timeStr).getTime()); + Date date = stringToDate(timeStr); + return null == date ? null : new Timestamp(date.getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { @@ -802,7 +815,8 @@ public static java.sql.Date getDateFromStr(String dateStr) { Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); } - return null == stringToDate(dateStr) ? null : new java.sql.Date(stringToDate(dateStr).getTime()); + Date date = stringToDate(dateStr); + return null == date ? null : new java.sql.Date(date.getTime()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 5aaa40a15..c5584b5d8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -23,18 +23,6 @@ import java.math.BigInteger; import java.sql.Date; import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; - - -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.TimeZone; -import java.util.regex.Pattern; - -import static java.time.format.DateTimeFormatter.ISO_INSTANT; /** * Convert val to specified numeric type diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java index d399b533c..dfbab87a7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java @@ -37,16 +37,23 @@ package com.dtstack.flink.sql.util; import com.google.common.collect.HashBasedTable; -import org.apache.calcite.sql.*; + +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.*; +import static org.apache.calcite.sql.SqlKind.AS; +import static org.apache.calcite.sql.SqlKind.IDENTIFIER; +import static org.apache.calcite.sql.SqlKind.JOIN; + /** * @Auther: jiangjunjie diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index b0ca0335f..ffce0c646 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -29,8 +29,6 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import java.sql.Timestamp; -import java.lang.Long; - /** * define watermarker * Date: 2018/6/29 diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 0d97f5995..23213e721 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -32,7 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index ed2931fff..63d26d5dd 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -20,7 +20,10 @@ package com.dtstack.flink.sql.side.hbase; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; @@ -34,7 +37,12 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +50,10 @@ import java.io.IOException; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.*; +import java.util.Calendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class HbaseAllReqRow extends BaseAllReqRow { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index cde8d1b1d..78d316c99 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -4,12 +4,10 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java index 692e208b5..5f852832b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -53,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.TimeZone; import java.util.stream.Collectors; @@ -287,6 +286,7 @@ private Object convertFlinkType(Schema schema, Object object) { case DOUBLE: case BOOLEAN: return object; + default: } throw new RuntimeException("Unsupported Avro type:" + schema); } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java index 4e57b6f2a..4e7c727d7 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -47,10 +47,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; -import java.util.Iterator; import java.util.Objects; -import java.util.stream.IntStream; -import java.util.stream.Stream; /** * Serialization schema that serializes an object of Flink types into a CSV bytes. @@ -132,9 +129,9 @@ public Builder setFieldDelimiter(char c) { public Builder setLineDelimiter(String delimiter) { Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + if (!("\n".equals(delimiter)) && !("\r".equals(delimiter)) && !("\r\n".equals(delimiter))) { throw new IllegalArgumentException( - "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); } this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); return this; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index c27eee376..1210aba40 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -19,7 +19,6 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index bee1865dd..19e0feddb 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -24,7 +24,6 @@ import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index ee3423b07..b64727ee3 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -23,7 +23,6 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 9f8917761..01c67cd56 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -21,18 +21,15 @@ import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.Map; import java.util.Properties; /** diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index e0e023586..eec398de0 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -23,7 +23,6 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index 0cb11da82..b1cc53fb7 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -23,7 +23,6 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index c39d8b8a9..551a00533 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -20,9 +20,7 @@ package com.dtstack.flink.sql.launcher; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.launcher.perjob.PerJobClusterClientBuilder; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; @@ -43,14 +41,9 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.net.URLDecoder; import java.util.LinkedList; import java.util.List; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 0e5089bc9..5dc74900e 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -32,14 +32,12 @@ import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index c7fa2398e..c1a32cb64 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -37,8 +37,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index d85a9990d..a1cf94000 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -44,13 +44,6 @@ import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 3f1f9f7e2..37f7f5952 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -35,8 +35,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; /** * Reason: diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java index 58e92c143..1d775adac 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlDialect.java @@ -20,7 +20,6 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.util.DtStringUtil; import java.util.Arrays; import java.util.Optional; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index 28811d2fd..a41cad5ef 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -25,7 +25,6 @@ import java.math.BigDecimal; import java.sql.Date; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.sql.Types; diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index bdee79f07..8e6a08791 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index a63d28607..1f4fbf13a 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -19,7 +19,9 @@ package com.dtstack.flink.sql.side.sqlserver; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; From 91e8880ff70bafc2864abdf6f84598ed85f911fb Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 3 Apr 2020 21:03:31 +0800 Subject: [PATCH 082/378] rm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2952551ff..40ff81133 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: stage: test script: - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - sh ci/sonar_notify.sh only: - v1.8.0_dev From 388e7f8b8397594c96822a69d5f4c565ff1a13da Mon Sep 17 00:00:00 2001 From: maqi <313328862@qq.com> Date: Sat, 4 Apr 2020 14:01:05 +0800 Subject: [PATCH 083/378] deal minicluster --- .../sql/environment/MyLocalStreamEnvironment.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index a8728c595..029a07f99 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -113,16 +113,13 @@ public JobExecutionResult execute(String jobName) throws Exception { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = null; - try { - exec = new MiniCluster(configBuilder.build()); + try (MiniCluster exec = new MiniCluster(configBuilder.build());) { exec.start(); - return exec.executeJobBlocking(jobGraph); - } finally { + JobExecutionResult jobExecutionResult = exec.executeJobBlocking(jobGraph); transformations.clear(); - if (null != exec) { - exec.closeAsync(); - } + return jobExecutionResult; + } catch (Exception e) { + throw new RuntimeException(e); } } } From 2079ce409b665d21ca4ecf04efc0869f1f507ecf Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 10:35:27 +0800 Subject: [PATCH 084/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B0=8Dcountdown?= =?UTF-8?q?=E7=9A=84=E6=8D=95=E7=8D=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index da6cb3876..19d78eccd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -119,19 +119,18 @@ private void connectWithRetry(Map inputParams, CRow input, Resul while(!finishFlag.get()){ CountDownLatch latch = new CountDownLatch(1); rdbSqlClient.getConnection(conn -> { - if(conn.failed()){ - if(failCounter.getAndIncrement() % 1000 == 0){ - logger.error("getConnection error", conn.cause()); - } - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(conn.cause()); - finishFlag.set(true); - } - latch.countDown(); - conn.result().close(); - return; - } try { + if(conn.failed()){ + if(failCounter.getAndIncrement() % 1000 == 0){ + logger.error("getConnection error", conn.cause()); + } + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ + resultFuture.completeExceptionally(conn.cause()); + finishFlag.set(true); + } + conn.result().close(); + return; + } CONN_STATUS.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); From 8e4196a3b5da27115ecfbf0b8ae9052851b83737 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 8 Apr 2020 10:45:18 +0800 Subject: [PATCH 085/378] fix conflict --- .../flink/sql/exec/ExecuteProcessHelper.java | 16 ++++------ .../dtstack/flink/sql/side/SideSQLParser.java | 1 + .../dtstack/flink/sql/side/SideSqlExec.java | 30 ++++++------------- .../flink/sql/table/AbstractTableParser.java | 4 +-- .../sql/side/oracle/OracleAsyncSideInfo.java | 4 +-- .../flink/sql/sink/oracle/OracleDialect.java | 8 ++--- .../flink/sql/sink/rdb/AbstractRdbSink.java | 9 +++--- 7 files changed, 27 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 1c0c2a702..8b0c9ec10 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -20,6 +20,11 @@ import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.parser.CreateFuncParser; +import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.FlinkPlanner; +import com.dtstack.flink.sql.parser.InsertSqlParser; +import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.parser.SqlTree; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,17 +68,6 @@ import org.apache.calcite.sql.SqlNode; import org.apache.commons.io.Charsets; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.StreamQueryConfig; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 61546bb60..2296f64c5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -48,6 +48,7 @@ import java.util.Set; import static org.apache.calcite.sql.SqlKind.IDENTIFIER; +import static org.apache.calcite.sql.SqlKind.LITERAL; /** * Parsing sql, obtain execution information dimension table diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 7a657263c..c41e2264a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -47,23 +47,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -71,7 +61,6 @@ import java.sql.Timestamp; import java.util.Arrays; -import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -168,8 +157,7 @@ public void exec(String sql, }else if (pollObj instanceof JoinInfo){ LOG.info("----------exec join info----------\n{}", pollObj.toString()); - preIsSideJoin = true; - joinFun(pollObj, localTableCache, sideTableMap, tableEnv, replaceInfoList); + joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -295,7 +283,7 @@ private Table getTableFromCache(Map localTableCache, String table * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -303,7 +291,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { + private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -311,7 +299,7 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -370,7 +358,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -386,7 +374,7 @@ private void joinFun(Object pollObj, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } @@ -395,9 +383,9 @@ private void joinFun(Object pollObj, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } -// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ -// throw new RuntimeException("ON condition must contain all equal fields!!!"); -// } + if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ + throw new RuntimeException("ON condition must contain all equal fields!!!"); + } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 136779cda..1f210cbf2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -112,12 +112,12 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ Class fieldClass = null; - TableInfo.FieldExtraInfo fieldExtraInfo = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new TableInfo.FieldExtraInfo(); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index d2c75c716..658020b1c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -56,7 +56,7 @@ public String wrapperPlaceholder(String fieldName) { String rpadFormat = "rpad(?, %d, ' ')"; if (StringUtils.contains(type.toLowerCase(), "char")) { - TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(rpadFormat, charLength); diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index a687aa012..3a320658c 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; @@ -41,7 +41,7 @@ public class OracleDialect implements JDBCDialect { private List fieldList; private List fieldTypeList; - private List fieldExtraInfoList; + private List fieldExtraInfoList; @Override public boolean canHandle(String url) { @@ -133,7 +133,7 @@ public String wrapperPlaceholder(String fieldName) { String type = fieldTypeList.get(pos); if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(RPAD_FORMAT, charLength); @@ -151,7 +151,7 @@ public void setFieldTypeList(List fieldTypeList) { this.fieldTypeList = fieldTypeList; } - public void setFieldExtraInfoList(List fieldExtraInfoList) { + public void setFieldExtraInfoList(List fieldExtraInfoList) { this.fieldExtraInfoList = fieldExtraInfoList; } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index e71d4f427..ee51c22d1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -20,9 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -84,20 +83,20 @@ public abstract class AbstractRdbSink implements RetractStreamTableSink, Se public List fieldList; public List fieldTypeList; - public List fieldExtraInfoList; + public List fieldExtraInfoList; public AbstractRdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbURL = rdbTableInfo.getUrl(); + this.dbUrl = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); From aa9845a1e93a37243dc9fa2abfb7ac1679a5c1fe Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 11:27:18 +0800 Subject: [PATCH 086/378] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A0=B9=E6=8D=AE?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=AC=A1=E6=95=B0=E5=92=8CjoinType=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/AbstractSideTableInfo.java | 16 +++--------- .../flink/sql/side/BaseAsyncReqRow.java | 2 +- .../sql/table/AbstractSideTableParser.java | 2 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 26 ++++++++++++++++--- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 0a6babc8a..71a309411 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -68,13 +68,11 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private int asyncTimeout=10000; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; - private boolean partitionedJoin = false; private String cacheMode="ordered"; - private Integer asyncFailMaxNum; + private Long asyncFailMaxNum; private List predicateInfoes = Lists.newArrayList(); @@ -153,19 +151,11 @@ public List getPredicateInfoes() { return predicateInfoes; } - public int getAsyncTimeoutNumLimit() { - return asyncTimeoutNumLimit; - } - - public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { - this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; - } - - public Integer getAsyncFailMaxNum(Integer defaultValue) { + public Long getAsyncFailMaxNum(Long defaultValue) { return Objects.isNull(asyncFailMaxNum) ? defaultValue : asyncFailMaxNum; } - public void setAsyncFailMaxNum(Integer asyncFailMaxNum) { + public void setAsyncFailMaxNum(Long asyncFailMaxNum) { this.asyncFailMaxNum = asyncFailMaxNum; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 8713b541a..bfe14c3ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -149,7 +149,7 @@ public void timeout(CRow input, ResultFuture resultFuture) throws Exceptio resultFuture.complete(null); return; } - if(timeOutNum > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Integer.MAX_VALUE)){ + if(timeOutNum > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); return; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 83a06d256..7df86b3cb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -116,7 +116,7 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ sideTableInfo.setAsyncFailMaxNum(asyncFailNum); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 19d78eccd..e115ed21c 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -30,6 +30,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.calcite.sql.JoinType; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -38,16 +39,19 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Int; import java.sql.Timestamp; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; /** @@ -81,6 +85,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); + private final static AtomicLong TIMOUT_NUM = new AtomicLong(0); + private Logger logger = LoggerFactory.getLogger(getClass()); public RdbAsyncReqRow(BaseSideInfo sideInfo) { @@ -114,7 +120,7 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul } private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { - AtomicInteger failCounter = new AtomicInteger(0); + AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ CountDownLatch latch = new CountDownLatch(1); @@ -124,8 +130,8 @@ private void connectWithRetry(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ logger.error("getConnection error", conn.cause()); } - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3)){ - resultFuture.completeExceptionally(conn.cause()); + if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ + outByJoinType(resultFuture, conn.cause()); finishFlag.set(true); } conn.result().close(); @@ -202,8 +208,12 @@ private void handleQuery(SQLConnection connection,Map inputParam JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { if (rs.failed()) { + if(TIMOUT_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + outByJoinType(resultFuture, rs.cause()); + return; + } LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.completeExceptionally(rs.cause()); + resultFuture.complete(null); return; } @@ -242,4 +252,12 @@ private void handleQuery(SQLConnection connection,Map inputParam }); } + private void outByJoinType(ResultFuture resultFuture, Throwable e){ + if(sideInfo.getJoinType() == JoinType.LEFT){ + resultFuture.complete(null); + return; + } + resultFuture.completeExceptionally(e); + } + } From caa8824a6bcd8e14f13dbd2c3b6ffa284dea5c04 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 11:29:11 +0800 Subject: [PATCH 087/378] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index e115ed21c..c86434528 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -21,17 +21,15 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; -import io.vertx.core.json.Json; +import com.google.common.collect.Lists; import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; -import com.google.common.collect.Lists; import org.apache.calcite.sql.JoinType; -import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; @@ -39,20 +37,14 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Int; import java.sql.Timestamp; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; /** * Date: 2018/11/26 From ac65b81d92c5ca181418e082a929a905acadac60 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 11:40:37 +0800 Subject: [PATCH 088/378] =?UTF-8?q?rowkey=E7=9A=84=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E6=8B=BC=E6=8E=A5=E5=BD=A2=E5=BC=8F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseOutputFormat.java | 22 ++++++++----------- .../flink/sql/sink/hbase/HbaseSink.java | 2 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 2 +- .../sql/sink/hbase/table/HbaseTableInfo.java | 6 ++--- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 1769fbda7..a3189b0a6 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -55,7 +55,7 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String host; private String zkParent; - private String[] rowkey; + private String rowkey; private String tableName; private String[] columnNames; private String updateMode; @@ -164,25 +164,21 @@ private Put getPutByRow(Row record) { } private String buildRowKey(Row record) { - List rowKeyValues = getRowKeyValues(record); + String rowKeyValues = getRowKeyValues(record); // all rowkey not null - if (rowKeyValues.size() != rowkey.length) { + if (StringUtils.isBlank(rowKeyValues)) { LOG.error("row key value must not null,record is ..{}", record); outDirtyRecords.inc(); return ""; } - return StringUtils.join(rowKeyValues, "-"); + return rowKeyValues; } - private List getRowKeyValues(Row record) { - List rowKeyValues = Lists.newArrayList(); + private String getRowKeyValues(Row record) { Map row = rowConvertMap(record); - for (String key : rowkey) { - RowKeyBuilder rowKeyBuilder = new RowKeyBuilder(); - rowKeyBuilder.init(key); - rowKeyValues.add(rowKeyBuilder.getRowKey(row)); - } - return rowKeyValues; + RowKeyBuilder rowKeyBuilder = new RowKeyBuilder(); + rowKeyBuilder.init(rowkey); + return rowKeyBuilder.getRowKey(row); } private Map rowConvertMap(Row record){ @@ -232,7 +228,7 @@ public HbaseOutputFormatBuilder setTable(String tableName) { return this; } - public HbaseOutputFormatBuilder setRowkey(String[] rowkey) { + public HbaseOutputFormatBuilder setRowkey(String rowkey) { format.rowkey = rowkey; return this; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 9308725a2..13bd98b70 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -51,7 +51,7 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String parent; protected String tableName; protected String updateMode; - protected String[] rowkey; + protected String rowkey; public HbaseSink() { // TO DO NOTHING diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 8d8996a23..bd1b7cac3 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -69,7 +69,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map columnNameFamily; @@ -90,11 +90,11 @@ public void setParent(String parent) { this.parent = parent; } - public String[] getRowkey() { + public String getRowkey() { return rowkey; } - public void setRowkey(String[] rowkey) { + public void setRowkey(String rowkey) { this.rowkey = rowkey; } From 5f30a74b9518d4d33239b8726ff4d027442ca097 Mon Sep 17 00:00:00 2001 From: maqi Date: Wed, 8 Apr 2020 14:00:06 +0800 Subject: [PATCH 089/378] Revert "Merge branch 'feat_1.8release3.10.x_mergeTest' into '1.8_release_3.10.x'" This reverts merge request !18 --- .gitlab-ci.yml | 10 - README.md | 29 +- .../side/cassandra/CassandraAllReqRow.java | 10 +- .../side/cassandra/CassandraAllSideInfo.java | 10 +- .../side/cassandra/CassandraAsyncReqRow.java | 11 +- .../cassandra/CassandraAsyncSideInfo.java | 18 +- .../cassandra/table/CassandraSideParser.java | 17 +- .../table/CassandraSideTableInfo.java | 4 +- .../sink/cassandra/CassandraOutputFormat.java | 4 +- .../sql/sink/cassandra/CassandraSink.java | 4 +- .../cassandra/table/CassandraSinkParser.java | 10 +- .../cassandra/table/CassandraTableInfo.java | 4 +- .../test/java/com/dtstack/flinkx/AppTest.java | 43 +- ci/sonar_notify.sh | 14 - .../side/clickhouse/ClickhouseAllReqRow.java | 17 +- .../clickhouse/ClickhouseAllSideInfo.java | 4 +- .../clickhouse/ClickhouseAsyncReqRow.java | 12 +- .../clickhouse/ClickhouseAsyncSideInfo.java | 4 +- .../table/ClickhouseSideParser.java | 6 +- .../sql/sink/clickhouse/ClickhouseSink.java | 9 +- .../table/ClickhouseSinkParser.java | 6 +- .../sql/sink/console/ConsoleOutputFormat.java | 6 +- .../flink/sql/sink/console/ConsoleSink.java | 4 +- .../sink/console/table/ConsoleSinkParser.java | 10 +- .../sink/console/table/ConsoleTableInfo.java | 4 +- .../sink/console/table/TablePrintUtil.java | 23 +- core/pom.xml | 10 +- .../main/java/com/dtstack/flink/sql/Main.java | 5 +- .../sql/classloader/ClassLoaderManager.java | 4 +- .../flink/sql/constrant/ConfigConstrant.java | 13 +- .../dtstack/flink/sql/enums/ColumnType.java | 63 +-- .../flink/sql/enums/ECacheContentType.java | 9 - .../dtstack/flink/sql/enums/ECacheType.java | 13 +- .../flink/sql/enums/EDatabaseType.java | 9 - .../flink/sql/enums/EPluginLoadMode.java | 6 - .../flink/sql/enums/EStateBackend.java | 9 - .../environment/StreamEnvConfigManager.java | 59 +-- .../flink/sql/exec/ExecuteProcessHelper.java | 76 ++-- .../dtstack/flink/sql/exec/ParamsInfo.java | 1 + .../format/SerializationMetricWrapper.java | 11 +- .../DtNestRowDeserializationSchema.java | 11 +- .../flink/sql/option/OptionParser.java | 21 +- ...putFormat.java => DtRichOutputFormat.java} | 2 +- .../flink/sql/parser/CreateFuncParser.java | 8 +- .../dtstack/flink/sql/parser/SqlParser.java | 12 +- .../com/dtstack/flink/sql/parser/SqlTree.java | 8 +- .../{BaseAllReqRow.java => AllReqRow.java} | 18 +- ...{BaseAsyncReqRow.java => AsyncReqRow.java} | 18 +- .../flink/sql/side/JoinNodeDealer.java | 10 +- .../flink/sql/side/ParserJoinField.java | 2 - .../side/{BaseSideInfo.java => SideInfo.java} | 22 +- .../flink/sql/side/SidePredicatesParser.java | 8 +- .../dtstack/flink/sql/side/SideSQLParser.java | 7 +- .../dtstack/flink/sql/side/SideSqlExec.java | 41 +- ...tSideTableInfo.java => SideTableInfo.java} | 33 +- .../flink/sql/side/StreamSideFactory.java | 10 +- ...stractSideCache.java => AbsSideCache.java} | 8 +- .../flink/sql/side/cache/LRUSideCache.java | 6 +- .../sql/side/operator/SideAsyncOperator.java | 16 +- .../operator/SideWithAllCacheOperator.java | 18 +- .../flink/sql/sink/IStreamSinkGener.java | 4 +- .../flink/sql/sink/StreamSinkFactory.java | 12 +- .../flink/sql/source/IStreamSourceGener.java | 4 +- .../flink/sql/source/StreamSourceFactory.java | 12 +- ...bleParser.java => AbsSideTableParser.java} | 53 +-- ...SourceParser.java => AbsSourceParser.java} | 16 +- ...ctTableParser.java => AbsTableParser.java} | 20 +- .../sql/table/ITableFieldDealHandler.java | 2 +- ...rceTableInfo.java => SourceTableInfo.java} | 2 +- ...{AbstractTableInfo.java => TableInfo.java} | 2 +- ...leInfoParser.java => TableInfoParser.java} | 20 +- ...getTableInfo.java => TargetTableInfo.java} | 2 +- .../com/dtstack/flink/sql/util/ClassUtil.java | 3 +- .../com/dtstack/flink/sql/util/DateUtil.java | 408 ++++++++++-------- .../dtstack/flink/sql/util/DtStringUtil.java | 8 +- .../com/dtstack/flink/sql/util/JDBCUtils.java | 4 +- .../com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../dtstack/flink/sql/util/PluginUtil.java | 10 +- ...arker.java => AbsCustomerWaterMarker.java} | 4 +- .../CustomerWaterMarkerForLong.java | 2 +- .../CustomerWaterMarkerForTimeStamp.java | 3 +- .../sql/watermarker/WaterMarkerAssigner.java | 8 +- .../sql/side/SidePredicatesParserTest.java | 4 +- .../flink/sql/side/db2/Db2AllReqRow.java | 15 +- .../flink/sql/side/db2/Db2AllSideInfo.java | 4 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 10 +- .../flink/sql/side/db2/Db2AsyncSideInfo.java | 4 +- .../sql/side/db2/table/Db2SideParser.java | 6 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 9 +- .../flink/sql/sink/db/table/DbSinkParser.java | 6 +- docs/kafkaSink.md | 223 ---------- docs/kafkaSource.md | 224 ++++++++-- docs/mongoSide.md | 5 +- docs/mongoSink.md | 7 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../Elasticsearch6AllReqRow.java | 21 +- .../Elasticsearch6AllSideInfo.java | 11 +- .../Elasticsearch6AsyncReqRow.java | 11 +- .../Elasticsearch6AsyncSideInfo.java | 13 +- .../table/Elasticsearch6SideParser.java | 8 +- .../table/Elasticsearch6SideTableInfo.java | 4 +- .../sql/side/elasticsearch6/util/Es6Util.java | 10 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 22 +- .../sql/side/hbase/HbaseAllSideInfo.java | 10 +- .../sql/side/hbase/HbaseAsyncReqRow.java | 14 +- .../sql/side/hbase/HbaseAsyncSideInfo.java | 10 +- ...deDealer.java => AbsRowKeyModeDealer.java} | 12 +- .../PreRowKeyModeDealerDealer.java | 12 +- .../rowkeydealer/RowKeyEqualModeDealer.java | 9 +- .../sql/side/hbase/Md5ReplaceOperator.java | 2 +- ...laceOperator.java => ReplaceOperator.java} | 4 +- .../sql/side/hbase/enums/EReplaceOpType.java | 9 +- .../sql/side/hbase/enums/EReplaceType.java | 9 - .../sql/side/hbase/table/HbaseSideParser.java | 12 +- .../side/hbase/table/HbaseSideTableInfo.java | 4 +- .../sql/side/hbase/utils/HbaseUtils.java | 2 - .../sql/sink/hbase/HbaseOutputFormat.java | 11 +- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 10 +- .../sql/sink/hbase/table/HbaseTableInfo.java | 4 +- .../sql/side/impala/ImpalaAllReqRow.java | 13 +- .../sql/side/impala/ImpalaAllSideInfo.java | 7 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 29 +- .../sql/side/impala/ImpalaAsyncSideInfo.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 6 +- .../flink/sql/sink/impala/ImpalaSink.java | 14 +- .../sink/impala/table/ImpalaSinkParser.java | 7 +- .../kafka/AbstractKafkaProducerFactory.java | 37 +- .../sql/sink/kafka/AbstractKafkaSink.java | 148 ------- .../sink/kafka/CustomerFlinkPartition.java | 8 +- .../CustomerKeyedSerializationSchema.java | 35 +- .../AvroCRowSerializationSchema.java | 366 ---------------- .../CsvCRowSerializationSchema.java | 374 ---------------- .../JsonCRowSerializationSchema.java | 234 ---------- .../sql/sink/kafka/table/KafkaSinkParser.java | 13 +- .../sink/kafka/table/KafkaSinkTableInfo.java | 44 +- .../sql/source/kafka/AbstractKafkaSource.java | 117 ----- .../KafkaDeserializationMetricWrapper.java | 10 +- .../source/kafka/table/KafkaSourceParser.java | 42 +- .../kafka/table/KafkaSourceTableInfo.java | 181 ++++---- .../flink/sql/sink/kafka/KafkaProducer.java | 6 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 118 ++++- .../flink/sql/source/kafka/KafkaSource.java | 78 +++- .../flink/sql/sink/kafka/KafkaProducer09.java | 5 +- .../sink/kafka/KafkaProducer09Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 124 +++++- .../flink/sql/source/kafka/KafkaSource.java | 76 +++- .../sql/sink/kafka/KafkaProducer010.java | 6 +- .../sink/kafka/KafkaProducer010Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 122 +++++- .../flink/sql/source/kafka/KafkaSource.java | 89 +++- .../sql/sink/kafka/KafkaProducer011.java | 6 +- .../sink/kafka/KafkaProducer011Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 121 +++++- .../source/kafka/KafkaConsumer011Factory.java | 2 +- .../flink/sql/source/kafka/KafkaSource.java | 92 +++- .../flink/sql/side/kudu/KuduAllReqRow.java | 24 +- .../flink/sql/side/kudu/KuduAllSideInfo.java | 10 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 26 +- .../sql/side/kudu/KuduAsyncSideInfo.java | 10 +- .../sql/side/kudu/table/KuduSideParser.java | 11 +- .../side/kudu/table/KuduSideTableInfo.java | 4 +- .../flink/sql/side/kudu/utils/KuduUtil.java | 1 - .../flink/sql/sink/kudu/KuduOutputFormat.java | 8 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 4 +- .../sql/sink/kudu/table/KuduSinkParser.java | 11 +- .../sql/sink/kudu/table/KuduTableInfo.java | 4 +- .../sql/launcher/ClusterClientFactory.java | 8 +- .../perjob/PerJobClusterClientBuilder.java | 3 +- .../sql/launcher/perjob/PerJobSubmitter.java | 1 + .../flink/sql/side/mongo/MongoAllReqRow.java | 62 ++- .../sql/side/mongo/MongoAllSideInfo.java | 10 +- .../sql/side/mongo/MongoAsyncReqRow.java | 83 ++-- .../sql/side/mongo/MongoAsyncSideInfo.java | 13 +- .../sql/side/mongo/table/MongoSideParser.java | 12 +- .../side/mongo/table/MongoSideTableInfo.java | 4 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 1 - .../sql/sink/mongo/MongoOutputFormat.java | 44 +- .../flink/sql/sink/mongo/MongoSink.java | 4 +- .../sql/sink/mongo/table/MongoSinkParser.java | 10 +- .../sql/sink/mongo/table/MongoTableInfo.java | 4 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 +++ .../flink/sql/side/mysql/MysqlAllReqRow.java | 12 +- .../sql/side/mysql/MysqlAllSideInfo.java | 4 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 15 +- .../sql/side/mysql/MysqlAsyncSideInfo.java | 4 +- .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../flink/sql/sink/mysql/MysqlSink.java | 6 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 6 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 +++ .../sql/side/oracle/OracleAllReqRow.java | 12 +- .../sql/side/oracle/OracleAllSideInfo.java | 5 +- .../sql/side/oracle/OracleAsyncReqRow.java | 15 +- .../sql/side/oracle/OracleAsyncSideInfo.java | 9 +- .../side/oracle/table/OracleSideParser.java | 6 +- .../flink/sql/sink/oracle/OracleDialect.java | 8 +- .../flink/sql/sink/oracle/OracleSink.java | 6 +- .../sink/oracle/table/OracleSinkParser.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 34 +- .../sql/side/polardb/PolardbAllSideInfo.java | 26 +- .../sql/side/polardb/PolardbAsyncReqRow.java | 32 +- .../side/polardb/PolardbAsyncSideInfo.java | 26 +- .../side/polardb/table/PolardbSideParser.java | 29 +- .../flink/sql/sink/polardb/PolardbSink.java | 30 +- .../sink/polardb/table/PolardbSinkParser.java | 28 +- pom.xml | 16 +- .../side/postgresql/PostgresqlAllReqRow.java | 12 +- .../postgresql/PostgresqlAllSideInfo.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 10 +- .../postgresql/PostgresqlAsyncSideInfo.java | 4 +- .../table/PostgresqlSideParser.java | 6 +- .../sql/sink/postgresql/PostgresqlSink.java | 6 +- .../table/PostgresqlSinkParser.java | 6 +- ...actRdbAllReqRow.java => RdbAllReqRow.java} | 20 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 15 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 45 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 16 +- .../provider/DTC3P0DataSourceProvider.java | 10 +- .../sql/side/rdb/table/RdbSideParser.java | 8 +- .../sql/side/rdb/table/RdbSideTableInfo.java | 15 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 1 - .../flink/sql/sink/rdb/JDBCOptions.java | 22 +- .../{AbstractRdbSink.java => RdbSink.java} | 19 +- .../rdb/format/AbstractJDBCOutputFormat.java | 14 +- .../rdb/format/JDBCUpsertOutputFormat.java | 14 +- .../rdb/format/RetractJDBCOutputFormat.java | 0 .../sql/sink/rdb/table/RdbSinkParser.java | 9 +- .../sql/sink/rdb/table/RdbTableInfo.java | 6 +- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 13 +- ...actUpsertWriter.java => UpsertWriter.java} | 86 ++-- .../flink/sql/side/redis/RedisAllReqRow.java | 32 +- .../sql/side/redis/RedisAllSideInfo.java | 14 +- .../sql/side/redis/RedisAsyncReqRow.java | 26 +- .../sql/side/redis/RedisAsyncSideInfo.java | 14 +- .../sql/side/redis/table/RedisSideParser.java | 11 +- .../sql/side/redis/table/RedisSideReqRow.java | 8 +- .../side/redis/table/RedisSideTableInfo.java | 8 +- .../sql/sink/redis/RedisOutputFormat.java | 29 +- .../flink/sql/sink/redis/RedisSink.java | 8 +- .../sql/sink/redis/table/RedisSinkParser.java | 12 +- .../sql/sink/redis/table/RedisTableInfo.java | 8 +- .../CustomerSocketTextStreamFunction.java | 6 +- .../serversocket/ServersocketSource.java | 4 +- .../table/ServersocketSourceParser.java | 8 +- .../table/ServersocketSourceTableInfo.java | 6 +- .../side/sqlserver/SqlserverAllReqRow.java | 20 +- .../side/sqlserver/SqlserverAllSideInfo.java | 10 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 14 +- .../sqlserver/SqlserverAsyncSideInfo.java | 11 +- .../sqlserver/table/SqlserverSideParser.java | 12 +- .../sql/sink/sqlserver/SqlserverSink.java | 6 +- .../sqlserver/table/SqlserverSinkParser.java | 6 +- 258 files changed, 2672 insertions(+), 3589 deletions(-) delete mode 100644 .gitlab-ci.yml rename kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java => cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java (56%) delete mode 100644 ci/sonar_notify.sh rename core/src/main/java/com/dtstack/flink/sql/outputformat/{AbstractDtRichOutputFormat.java => DtRichOutputFormat.java} (95%) rename core/src/main/java/com/dtstack/flink/sql/side/{BaseAllReqRow.java => AllReqRow.java} (79%) rename core/src/main/java/com/dtstack/flink/sql/side/{BaseAsyncReqRow.java => AsyncReqRow.java} (89%) rename core/src/main/java/com/dtstack/flink/sql/side/{BaseSideInfo.java => SideInfo.java} (92%) rename core/src/main/java/com/dtstack/flink/sql/side/{AbstractSideTableInfo.java => SideTableInfo.java} (79%) rename core/src/main/java/com/dtstack/flink/sql/side/cache/{AbstractSideCache.java => AbsSideCache.java} (84%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractSideTableParser.java => AbsSideTableParser.java} (62%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractSourceParser.java => AbsSourceParser.java} (81%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTableParser.java => AbsTableParser.java} (88%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractSourceTableInfo.java => SourceTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTableInfo.java => TableInfo.java} (98%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTableInfoParser.java => TableInfoParser.java} (83%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbstractTargetTableInfo.java => TargetTableInfo.java} (94%) rename core/src/main/java/com/dtstack/flink/sql/watermarker/{AbstractCustomerWaterMarker.java => AbsCustomerWaterMarker.java} (94%) delete mode 100644 docs/kafkaSink.md rename hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/{AbstractRowKeyModeDealer.java => AbsRowKeyModeDealer.java} (87%) rename hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/{AbstractReplaceOperator.java => ReplaceOperator.java} (93%) delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java delete mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java delete mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java create mode 100644 mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java create mode 100644 mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java rename rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/{AbstractRdbAllReqRow.java => RdbAllReqRow.java} (92%) rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/{AbstractRdbSink.java => RdbSink.java} (90%) delete mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/{AbstractUpsertWriter.java => UpsertWriter.java} (84%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 2952551ff..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,10 +0,0 @@ -build: - stage: test - script: - - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q - - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar - - sh ci/sonar_notify.sh - only: - - v1.8.0_dev - tags: - - dt-insight-engine \ No newline at end of file diff --git a/README.md b/README.md index c84d77d5a..db25164b6 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,34 @@ > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus + ## 新特性: + * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 + * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 + * 3.异步维表支持非等值连接，比如：<>,<,>。 + * 4.增加kafka数组解析 + * 5.增加kafka1.0以上版本的支持 + * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 + * 7.支持插件的依赖方式,参考pluginLoadMode参数 + * 8.支持cep处理 + * 9.支持udaf + * 10.支持谓词下移 + * 11.支持状态的ttl + + ## BUG修复： + * 1.修复不能解析sql中orderby,union语法。 + * 2.修复yarnPer模式提交失败的异常。 + * 3.一些bug的修复 + # 已支持 * 源表：kafka 0.9、0.10、0.11、1.x版本 * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse, impala, db2, sqlserver * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse, impala, db2, sqlserver +# 后续开发计划 + * 维表快照 + * kafka avro格式 + * topN + ## 1 快速起步 ### 1.1 运行模式 @@ -126,10 +149,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * restore.enable：是否失败重启（默认是true） - * failure.interval：衡量失败率的时间段，单位分钟（默认6m） - * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） - * logLevel: 日志级别动态配置（默认info） + * logLevel: 日志级别动态配置（默认info） * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 @@ -182,7 +202,6 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * [impala 结果表插件](docs/impalaSink.md) * [db2 结果表插件](docs/db2Sink.md) * [sqlserver 结果表插件](docs/sqlserverSink.md) -* [kafka 结果表插件](docs/kafkaSink.md) ### 2.3 维表插件 * [hbase 维表插件](docs/hbaseSide.md) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 3ad8b934e..bc0a0f8cd 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -34,10 +34,10 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -62,12 +62,14 @@ * * @author xuqianjin */ -public class CassandraAllReqRow extends BaseAllReqRow { +public class CassandraAllReqRow extends AllReqRow { private static final long serialVersionUID = 54015343561288219L; private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class); + private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver"; + private static final int CONN_RETRY_NUM = 3; private static final int FETCH_SIZE = 1000; @@ -77,7 +79,7 @@ public class CassandraAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index 5d95dc9bf..fa665f9a0 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -37,16 +37,16 @@ * * @author xuqianjin */ -public class CassandraAllSideInfo extends BaseSideInfo { +public class CassandraAllSideInfo extends SideInfo { private static final long serialVersionUID = -8690814317653033557L; - public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; sqlCondition = "select ${selectField} from ${tableName} "; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 7eee3cb5f..0173f2d6f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -37,11 +37,11 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; @@ -67,7 +67,7 @@ * * @author xuqianjin */ -public class CassandraAsyncReqRow extends BaseAsyncReqRow { +public class CassandraAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = 6631584128079864735L; @@ -83,7 +83,7 @@ public class CassandraAsyncReqRow extends BaseAsyncReqRow { private transient ListenableFuture session; private transient CassandraSideTableInfo cassandraSideTableInfo; - public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -216,7 +216,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce connCassandraDB(cassandraSideTableInfo); String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - LOG.info("sqlCondition:{}" + sqlCondition); + System.out.println("sqlCondition:" + sqlCondition); ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -265,6 +265,7 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); + System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 4bee5648b..3557f0f73 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -30,8 +30,6 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -41,18 +39,16 @@ * * @author xuqianjin */ -public class CassandraAsyncSideInfo extends BaseSideInfo { +public class CassandraAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -4403313049809013362L; - private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName()); - - public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -67,9 +63,9 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf } sqlCondition = "select ${selectField} from ${tableName}"; - sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - LOG.info("---------side_exe_sql-----\n{}" + sqlCondition); + sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index e893e56b0..62dd753b9 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -19,16 +19,18 @@ package com.dtstack.flink.sql.side.cassandra.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; +import java.math.BigDecimal; +import java.sql.Date; import java.sql.Timestamp; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -36,7 +38,7 @@ * * @author xuqianjin */ -public class CassandraSideParser extends AbstractSideTableParser { +public class CassandraSideParser extends AbsSideTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; @@ -71,7 +73,7 @@ public CassandraSideParser() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo(); cassandraSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraSideTableInfo); @@ -94,10 +96,9 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class CassandraOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index 26152a7d3..eb7b23b53 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.cassandra.table.CassandraTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,7 +63,7 @@ public CassandraSink() { } @Override - public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public CassandraSink genStreamSink(TargetTableInfo targetTableInfo) { CassandraTableInfo cassandraTableInfo = (CassandraTableInfo) targetTableInfo; this.address = cassandraTableInfo.getAddress(); this.tableName = cassandraTableInfo.getTableName(); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java index 9ef8639ba..4c68e71ae 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +33,7 @@ * * @author xuqianjin */ -public class CassandraSinkParser extends AbstractTableParser { +public class CassandraSinkParser extends AbsTableParser { public static final String ADDRESS_KEY = "address"; @@ -60,7 +60,7 @@ public class CassandraSinkParser extends AbstractTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { CassandraTableInfo cassandraTableInfo = new CassandraTableInfo(); cassandraTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraTableInfo); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index ffb5fa876..c6626c42a 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class CassandraTableInfo extends AbstractTargetTableInfo { +public class CassandraTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "cassandra"; diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java b/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java similarity index 56% rename from kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java rename to cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java index 476ccc8b7..33a0233ac 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java +++ b/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java @@ -16,16 +16,43 @@ * limitations under the License. */ -package com.dtstack.flink.sql.source.kafka.enums; + + +package com.dtstack.flinkx; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; /** - * Date: 2020/3/20 - * Company: www.dtstack.com - * @author maqi + * Unit test for simple App. */ -public enum EKafkaOffset { +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } - LATEST, - EARLIEST, - NONE + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } } diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh deleted file mode 100644 index 41f8a3c0e..000000000 --- a/ci/sonar_notify.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -#参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq - sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) - curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ - -H "Content-Type: application/json" \ - -d "{ - \"msgtype\": \"markdown\", - \"markdown\": { - \"title\":\"sonar代码质量\", - \"text\": \"## sonar代码质量报告: \n -> [sonar地址](http://172.16.100.198:9000/dashboard?id=dt-insight-engine/flinkStreamSQL) \n -> ${sonarreport} \n\" - } - }" \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index c9a0c447b..68c0c7984 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -20,37 +20,40 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; import java.util.List; +import java.util.Map; -public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { +public class ClickhouseAllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String passWord) { + public Connection getConn(String dbURL, String userName, String passWord) { try { Connection connection ; JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); // ClickHouseProperties contains all properties if (userName == null) { - connection = DriverManager.getConnection(dbUrl); + connection = DriverManager.getConnection(dbURL); } else { - connection = DriverManager.getConnection(dbUrl, userName, passWord); + connection = DriverManager.getConnection(dbURL, userName, passWord); } return connection; } catch (Exception e) { diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java index 43fbeaa56..973c069b9 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -28,7 +28,7 @@ public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 9da7d8eaa..2b5223412 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -37,7 +37,7 @@ public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -48,7 +48,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -58,11 +58,11 @@ public void open(Configuration parameters) throws Exception { System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(rdbSideTableInfo.getAsyncPoolSize()); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java index eec5fbe74..254561de0 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -29,7 +29,7 @@ public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java index ab285c37f..7be387fd8 100644 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.clickhouse.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -38,8 +38,8 @@ public class ClickhouseSideParser extends RdbSideParser { private static final String CURR_TYPE = "clickhouse"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index 5f7e41f7b..bbb6a6fe4 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,11 +22,14 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; +import java.util.List; +import java.util.Map; -public class ClickhouseSink extends AbstractRdbSink implements IStreamSinkGener { + +public class ClickhouseSink extends RdbSink implements IStreamSinkGener { public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -34,7 +37,7 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 5b0f2598f..8c3df93d7 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -30,8 +30,8 @@ public class ClickhouseSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "clickhouse"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index fc5a768c4..a397036ef 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends AbstractDtRichOutputFormat { +public class ConsoleOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); @@ -56,7 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { - LOG.info("received oriainal data:{}" + tuple2); + System.out.println("received oriainal data:" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java index 6cfdd2d3d..77a3efea2 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.console; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -80,7 +80,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ConsoleSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { return this; } } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java index 93ed02420..e77444bfd 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java @@ -18,13 +18,13 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -32,9 +32,9 @@ * * @author xuqianjin */ -public class ConsoleSinkParser extends AbstractTableParser { +public class ConsoleSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); consoleTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, consoleTableInfo); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java index 6fd7063c6..4b286c667 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; /** * Reason: @@ -26,7 +26,7 @@ * * @author xuqianjin */ -public class ConsoleTableInfo extends AbstractTargetTableInfo { +public class ConsoleTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "console"; diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 12bb2a5d4..8813da619 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -22,7 +22,6 @@ public class TablePrintUtil { public static final int ALIGN_LEFT = 1;//左对齐 public static final int ALIGN_RIGHT = 2;//右对齐 public static final int ALIGN_CENTER = 3;//居中对齐 - private static final Pattern PATTERN = Pattern.compile("[\u4e00-\u9fa5]"); private int align = ALIGN_CENTER;//默认居中对齐 private boolean equilong = false;//默认不等宽 @@ -59,9 +58,7 @@ public static TablePrintUtil build(String[][] data) { public static TablePrintUtil build(List data) { TablePrintUtil self = new TablePrintUtil(); self.data = new ArrayList<>(); - if (data.size() <= 0) { - throw new RuntimeException("数据源至少得有一行吧"); - } + if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); Object obj = data.get(0); @@ -73,9 +70,7 @@ public static TablePrintUtil build(List data) { int length = ((List) obj).size(); for (Object item : data) { List col = (List) item; - if (col.size() != length) { - throw new RuntimeException("数据源每列长度必须一致"); - } + if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); self.data.add(col.toArray(new String[length])); } } else { @@ -95,7 +90,7 @@ public static TablePrintUtil build(List data) { try { value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - LOG.error("", e); + e.printStackTrace(); } item[j] = value == null ? "null" : value; } @@ -120,7 +115,7 @@ private static List

getColList(Object obj) { Method[] methods = obj.getClass().getMethods(); for (Method m : methods) { StringBuilder getMethodName = new StringBuilder(m.getName()); - if ("get".equals(getMethodName.substring(0, 3)) && !"getClass".equals(m.getName())) { + if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { Col col = new Col(); col.getMethodName = getMethodName.toString(); char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); @@ -139,8 +134,8 @@ private static ListgetColList(Object obj) { * @return */ private int getStringCharLength(String str) { - //利用正则找到中文 - Matcher m = PATTERN.matcher(str); + Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 + Matcher m = p.matcher(str); int count = 0; while (m.find()) { count++; @@ -168,9 +163,7 @@ private int[] getColLengths() { if (equilong) {//如果等宽表格 int max = 0; for (int len : result) { - if (len > max) { - max = len; - } + if (len > max) max = len; } for (int i = 0; i < result.length; i++) { result[i] = max; @@ -220,8 +213,6 @@ public String getTableString() { sb.append(cell); for (int i = 0; i < right + padding; i++) {sb.append(s);} break; - default: - break; } sb.append(v); } diff --git a/core/pom.xml b/core/pom.xml index eb470a3c0..b17209612 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ 1.16.0 2.7.9 19.0 - 1.0.0-SNAPSHOT + 1.1.7 @@ -122,11 +122,13 @@ junit 4.12 + - com.aiweiergou - tools-logger - ${logger.tool.version} + ch.qos.logback + logback-classic + ${logback.version} + diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index fe925f9a6..b54c3feb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -21,12 +21,14 @@ +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Date: 2018/6/26 * Company: www.dtstack.com @@ -38,7 +40,6 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); - ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index 2e62e11ab..c03f60617 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -98,9 +98,9 @@ public static List getClassPath() { - public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) + public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) throws IllegalAccessException, InvocationTargetException { - for(URL url : jarUrlList){ + for(URL url : jarURLList){ if(url.toString().endsWith(".jar")){ urlClassLoaderAddUrl(classLoader, url); } diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 13e1de8c8..00d1ea0a5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,21 +63,14 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; - public static final String RESTOREENABLE = "restore.enable"; - public static final String LOG_LEVEL_KEY = "logLevel"; // restart plocy - public static final int FAILUEE_RATE = 3; - - public static final int FAILUEE_INTERVAL = 6; //min - - public static final int DELAY_INTERVAL = 10; //sec - - public static final String FAILUREINTERVAL = "failure.interval"; //min + public static final int failureRate = 3; - public static final String DELAYINTERVAL= "delay.interval"; //sec + public static final int failureInterval = 6; //min + public static final int delayInterval = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 7f3f0019c..749bbc907 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -46,66 +46,11 @@ * @author huyifan.zju@163.com */ public enum ColumnType { - /* - * string - */ - STRING, - /** - * varchar - */ - VARCHAR, - /** - * char - */ - CHAR, - /** - * int - */ - INT, - /** - * mediumint - */ - MEDIUMINT, - /** - * tinyint - */ - TINYINT, - /** - * datetime - */ - DATETIME, - /** - * smallint - */ - SMALLINT, - /** - * bigint - */ - BIGINT, - /** - * double - */ - DOUBLE, - /** - * float - */ - FLOAT, - /** - * boolean - */ + STRING, VARCHAR, CHAR, + INT, MEDIUMINT, TINYINT, DATETIME, SMALLINT, BIGINT, + DOUBLE, FLOAT, BOOLEAN, - /** - * date - */ - DATE, - /** - * timestamp - */ - TIMESTAMP, - /** - * decimal - */ - DECIMAL; + DATE, TIMESTAMP, DECIMAL; public static ColumnType fromString(String type) { if(type == null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java index 49e352757..66160d820 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java @@ -30,17 +30,8 @@ public enum ECacheContentType { - /** - * 无 - */ MissVal(0), - /** - * 1行 - */ SingleLine(1), - /** - * 多行 - */ MultiLine(2); int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java index 9d5bb5d11..582148c2c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java @@ -27,18 +27,7 @@ * @author xuchao */ public enum ECacheType { - /** - * none - */ - NONE, - /** - * lru - */ - LRU, - /** - * all - */ - ALL; + NONE, LRU, ALL; public static boolean isValid(String type){ for(ECacheType tmpType : ECacheType.values()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java index 7d2235f2e..9b01bf052 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java @@ -26,17 +26,8 @@ */ public enum EDatabaseType { - /** - * mysql - */ MYSQL, - /** - * sqlserver - */ SQLSERVER, - /** - * oracle - */ ORACLE, } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java index 439966dd2..6cb027ac3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java @@ -26,13 +26,7 @@ */ public enum EPluginLoadMode { - /** - * 0:classpath - */ CLASSPATH(0), - /** - * 1:shipfile - */ SHIPFILE(1); private int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java index 098cb57fe..a8f926175 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java @@ -25,17 +25,8 @@ * @author maqi */ public enum EStateBackend { - /** - * memory - */ MEMORY, - /** - * rockdb - */ ROCKSDB, - /** - * filesystem - */ FILESYSTEM; public static EStateBackend convertFromString(String type) { diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index d7b772c6e..ce763ea99 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,12 +18,6 @@ package com.dtstack.flink.sql.environment; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.EStateBackend; -import com.dtstack.flink.sql.util.MathUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -40,6 +34,13 @@ import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.java.StreamTableEnvironment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -101,15 +102,11 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - if(isRestore(confProperties).get()){ - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.FAILUEE_RATE, - Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), - Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) - )); - } else { - streamEnv.setRestartStrategy(RestartStrategies.noRestart()); - } + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.failureRate, + Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), + Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + )); // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -167,20 +164,6 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } - public static Optional isRestore(Properties properties){ - String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); - return Optional.of(Boolean.valueOf(restoreEnable)); - } - - public static Optional getDelayInterval(Properties properties){ - String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); - return Optional.of(Integer.valueOf(delayInterval)); - } - public static Optional getFailureInterval(Properties properties){ - String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); - return Optional.of(Integer.valueOf(failureInterval)); - } - /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties @@ -276,8 +259,6 @@ private static Optional createStateBackend(String backendType, Str checkpointDataUriEmptyCheck(checkpointDataUri, backendType); stateBackend = new RocksDBStateBackend(checkpointDataUri, BooleanUtils.toBoolean(backendIncremental)); break; - default: - break; } return stateBackend == null ? Optional.empty() : Optional.of(stateBackend); } @@ -337,14 +318,14 @@ private static void verityTtl(String ttlMintimeStr, String ttlMaxtimeStr) { * @return */ private static Long getTtlTime(Integer timeNumber, String timeUnit) { - if ("d".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L * 60 * 60 * 24; - } else if ("h".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L * 60 * 60; - } else if ("m".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L * 60; - } else if ("s".equalsIgnoreCase(timeUnit)) { - return timeNumber * 1000L; + if (timeUnit.equalsIgnoreCase("d")) { + return timeNumber * 1000l * 60 * 60 * 24; + } else if (timeUnit.equalsIgnoreCase("h")) { + return timeNumber * 1000l * 60 * 60; + } else if (timeUnit.equalsIgnoreCase("m")) { + return timeNumber * 1000l * 60; + } else if (timeUnit.equalsIgnoreCase("s")) { + return timeNumber * 1000l; } else { throw new RuntimeException("not support " + timeNumber + timeUnit); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 8b0c9ec10..823cd5143 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,13 +18,7 @@ package com.dtstack.flink.sql.exec; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; -import com.dtstack.flink.sql.parser.CreateFuncParser; -import com.dtstack.flink.sql.parser.CreateTmpTableParser; -import com.dtstack.flink.sql.parser.FlinkPlanner; -import com.dtstack.flink.sql.parser.InsertSqlParser; -import com.dtstack.flink.sql.parser.SqlParser; -import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.parser.*; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +33,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -49,12 +42,12 @@ import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; @@ -97,8 +90,11 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { LOG.info("------------program params-------------------------"); + System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); + Arrays.stream(args).forEach(System.out::println); LOG.info("-------------------------------------------"); + System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); Options options = optionParser.getOptions(); @@ -109,14 +105,13 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); - String logLevel = options.getLogLevel(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - List jarUrlList = getExternalJarUrls(options.getAddjar()); + List jarURList = getExternalJarUrls(options.getAddjar()); return ParamsInfo.builder() .setSql(sql) @@ -126,7 +121,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarUrlList) + .setJarUrlList(jarURList) .build(); } @@ -155,7 +150,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); - Map sideTableMap = Maps.newHashMap(); + Map sideTableMap = Maps.newHashMap(); Map registerTableCache = Maps.newHashMap(); //register udf @@ -188,14 +183,14 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - + private static void sqlTranslation(String localSqlPluginPath, - StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, - Map registerTableCache, - StreamQueryConfig queryConfig) throws Exception { + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); @@ -227,8 +222,9 @@ private static void sqlTranslation(String localSqlPluginPath, //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { - LOG.info("----------exec sql without dimension join-----------"); - LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); + System.out.println("----------exec sql without dimension join-----------"); + System.out.println("----------real sql exec is--------------------------"); + System.out.println(result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { LOG.info("exec sql: " + result.getExecSql()); @@ -268,14 +264,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @throws Exception */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { Set pluginClassPatshSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - if (tableInfo instanceof AbstractSourceTableInfo) { + if (tableInfo instanceof SourceTableInfo) { - AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); //Note --- parameter conversion function can not be used inside a function of the type of polymerization @@ -306,21 +302,21 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof AbstractTargetTableInfo) { + } else if (tableInfo instanceof TargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); + TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof AbstractSideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); + } else if (tableInfo instanceof SideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); @@ -353,12 +349,4 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti } - public static void setLogLevel(ParamsInfo paramsInfo){ - String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 27cc7702d..b36dc31eb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -84,6 +84,7 @@ public Properties getConfProp() { return confProp; } + @Override public String toString() { return "ParamsInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 8802198a0..3a5af18b1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -24,7 +24,6 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; @@ -35,11 +34,11 @@ * author: toutian * create: 2019/12/24 */ -public class SerializationMetricWrapper implements SerializationSchema { +public class SerializationMetricWrapper implements SerializationSchema { private static final long serialVersionUID = 1L; - private SerializationSchema serializationSchema; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; @@ -48,7 +47,7 @@ public class SerializationMetricWrapper implements SerializationSchema { protected transient Meter dtNumRecordsOutRate; - public SerializationMetricWrapper(SerializationSchema serializationSchema) { + public SerializationMetricWrapper(SerializationSchema serializationSchema) { this.serializationSchema = serializationSchema; } @@ -58,7 +57,7 @@ public void initMetric() { } @Override - public byte[] serialize(CRow element) { + public byte[] serialize(Row element) { beforeSerialize(); byte[] row = serializationSchema.serialize(element); afterSerialize(); @@ -80,7 +79,7 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } - public SerializationSchema getSerializationSchema() { + public SerializationSchema getSerializationSchema() { return serializationSchema; } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 5f1b1c6f3..78b579305 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.format.dtnest; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -28,9 +28,8 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import java.io.IOException; @@ -58,9 +57,9 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -76,7 +75,7 @@ public Row deserialize(byte[] message) throws IOException { try { for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 0fd057c5a..43f599d14 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -31,10 +31,7 @@ import java.io.File; import java.io.FileInputStream; import java.net.URLEncoder; -import java.util.stream.Stream; - import org.apache.commons.codec.Charsets; -import org.apache.flink.util.FileUtils; /** @@ -95,16 +92,19 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.objectToMap(properties); + Map mapConf = PluginUtil.ObjectToMap(properties); List args = Lists.newArrayList(); - for (Map.Entry one : mapConf.entrySet()) { + for(Map.Entry one : mapConf.entrySet()){ String key = one.getKey(); Object value = one.getValue(); - if (value == null) { + if(value == null){ continue; - } else if (OPTION_SQL.equalsIgnoreCase(key)) { + }else if(OPTION_SQL.equalsIgnoreCase(key)){ File file = new File(value.toString()); - String content = FileUtils.readFile(file, "UTF-8"); + FileInputStream in = new FileInputStream(file); + byte[] filecontent = new byte[(int) file.length()]; + in.read(filecontent); + String content = new String(filecontent, Charsets.UTF_8.name()); value = URLEncoder.encode(content, Charsets.UTF_8.name()); } args.add("-" + key); @@ -112,4 +112,9 @@ public List getProgramExeArgList() throws Exception { } return args; } + + public static void main(String[] args) throws Exception { + OptionParser OptionParser = new OptionParser(args); + System.out.println(OptionParser.getOptions()); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java rename to core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java index fbcc86bbd..1fc40c13b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java @@ -27,7 +27,7 @@ * extend RichOutputFormat with metric 'dtNumRecordsOut', 'dtNumDirtyRecordsOut', 'dtNumRecordsOutRate' * Created by sishu.yss on 2018/11/28. */ -public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ +public abstract class DtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index fc6a7e16c..670d98a7e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -32,18 +32,18 @@ public class CreateFuncParser implements IParser { - private static final String FUNC_PATTERN_STR = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; - private static final Pattern FUNC_PATTERN = Pattern.compile(FUNC_PATTERN_STR); + private static final Pattern funcPattern = Pattern.compile(funcPatternStr); @Override public boolean verify(String sql) { - return FUNC_PATTERN.matcher(sql).find(); + return funcPattern.matcher(sql).find(); } @Override public void parseSql(String sql, SqlTree sqlTree) { - Matcher matcher = FUNC_PATTERN.matcher(sql); + Matcher matcher = funcPattern.matcher(sql); if(matcher.find()){ String type = matcher.group(1); String funcName = matcher.group(2); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 2afc76c48..a76c1b31a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfoParser; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; @@ -75,7 +75,7 @@ public static SqlTree parseSql(String sql) throws Exception { List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); SqlTree sqlTree = new SqlTree(); - AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); + TableInfoParser tableInfoParser = new TableInfoParser(); for(String childSql : sqlArr){ if(Strings.isNullOrEmpty(childSql)){ continue; @@ -113,7 +113,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -126,7 +126,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -144,7 +144,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } } else { - AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 5252ee022..1b64b7c68 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.google.common.collect.Maps; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class SqlTree { private Map preDealTableMap = Maps.newHashMap(); - private Map tableInfoMap = Maps.newLinkedHashMap(); + private Map tableInfoMap = Maps.newLinkedHashMap(); private List execSqlList = Lists.newArrayList(); @@ -89,11 +89,11 @@ public List getTmpSqlList(){ return tmpSqlList; } - public Map getTableInfoMap() { + public Map getTableInfoMap() { return tableInfoMap; } - public void addTableInfo(String tableName, AbstractTableInfo tableInfo){ + public void addTableInfo(String tableName, TableInfo tableInfo){ tableInfoMap.put(tableName, tableInfo); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index 3e3d0a8fb..64fd844b1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -25,15 +25,13 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.dtstack.flink.sql.factory.DTThreadFactory; import org.apache.calcite.sql.JoinType; import java.sql.SQLException; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -43,15 +41,13 @@ * @author xuchao */ -public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); - - protected BaseSideInfo sideInfo; + protected SideInfo sideInfo; private ScheduledExecutorService es; - public BaseAllReqRow(BaseSideInfo sideInfo){ + public AllReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -64,11 +60,11 @@ public BaseAllReqRow(BaseSideInfo sideInfo){ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); - LOG.info("----- all cacheRef init end-----"); + System.out.println("----- all cacheRef init end-----"); //start reload cache thread - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); - es = new ScheduledThreadPoolExecutor(1,new DTThreadFactory("cache-all-reload")); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java similarity index 89% rename from core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index e5bce4437..fae0ba9aa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; @@ -30,12 +30,14 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; +import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; +import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -45,17 +47,17 @@ * @author xuchao */ -public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); +public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; - protected BaseSideInfo sideInfo; + protected SideInfo sideInfo; protected transient Counter parseErrorRecords; - public BaseAsyncReqRow(BaseSideInfo sideInfo){ + public AsyncReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -67,12 +69,12 @@ public void open(Configuration parameters) throws Exception { } private void initCache(){ - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ return; } - AbstractSideCache sideCache; + AbsSideCache sideCache; if(ECacheType.LRU.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ sideCache = new LRUSideCache(sideTableInfo); sideInfo.setSideCache(sideCache); @@ -121,10 +123,10 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { @Override public void timeout(CRow input, ResultFuture resultFuture) throws Exception { + //TODO 需要添加数据指标 if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } - timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index e38c263e2..f072e2591 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; -import com.esotericsoftware.minlog.Log; import com.google.common.base.Preconditions; import com.google.common.collect.*; import org.apache.calcite.sql.JoinType; @@ -440,18 +439,18 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, tableRef.put(tbTmp, tableAlias); } - Log.info("-------build temporary query-----------\n{}", tmpSelectSql); - Log.info("---------------------------------------"); + System.out.println("-------build temporary query-----------"); + System.out.println(tmpSelectSql); + System.out.println("---------------------------------------"); }catch (Exception e){ - Log.error("", e); + e.printStackTrace(); throw new RuntimeException(e); } } /** * 抽取上层需用使用到的字段 - * 由于where字段已经抽取到上一层了所以不用查询出来 * @param parentSelectList * @param fromTableNameSet * @return @@ -593,6 +592,7 @@ private void extractSelectField(SqlNode selectNode, } }else if(selectNode.getKind() == CASE){ + System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index d9d79f34e..738e0f84e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -80,7 +80,6 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b switch(identifierSize) { case 1: fieldInfoList.addAll(getAllField(scope)); - break; default: SqlIdentifier tableIdentify = identifier.skipLast(1); JoinScope.ScopeChild scopeChild = scope.getScope(tableIdentify.getSimple()); @@ -100,7 +99,6 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b fieldInfo.setTypeInformation(type); fieldInfoList.add(fieldInfo); } - break; } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java similarity index 92% rename from core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index 3b2faa713..029c86e25 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class BaseSideInfo implements Serializable{ +public abstract class SideInfo implements Serializable{ protected RowTypeInfo rowTypeInfo; @@ -68,12 +68,12 @@ public abstract class BaseSideInfo implements Serializable{ //key:Returns the value of the position, value: the ref field name​in the side table protected Map sideFieldNameIndex = Maps.newHashMap(); - protected AbstractSideTableInfo sideTableInfo; + protected SideTableInfo sideTableInfo; - protected AbstractSideCache sideCache; + protected AbsSideCache sideCache; - public BaseSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, - AbstractSideTableInfo sideTableInfo){ + public SideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, + SideTableInfo sideTableInfo){ this.rowTypeInfo = rowTypeInfo; this.outFieldInfoList = outFieldInfoList; this.joinType = joinInfo.getJoinType(); @@ -172,7 +172,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } - public abstract void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo); + public abstract void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo); public RowTypeInfo getRowTypeInfo() { return rowTypeInfo; @@ -246,19 +246,19 @@ public void setSideFieldIndex(Map sideFieldIndex) { this.sideFieldIndex = sideFieldIndex; } - public AbstractSideTableInfo getSideTableInfo() { + public SideTableInfo getSideTableInfo() { return sideTableInfo; } - public void setSideTableInfo(AbstractSideTableInfo sideTableInfo) { + public void setSideTableInfo(SideTableInfo sideTableInfo) { this.sideTableInfo = sideTableInfo; } - public AbstractSideCache getSideCache() { + public AbsSideCache getSideCache() { return sideCache; } - public void setSideCache(AbstractSideCache sideCache) { + public void setSideCache(AbsSideCache sideCache) { this.sideCache = sideCache; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 4524145db..0902bf39f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -45,7 +45,7 @@ * @author maqi */ public class SidePredicatesParser { - public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { + public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); @@ -57,7 +57,7 @@ public void fillPredicatesForSideTable(String exeSql, Map sideTableMap, Map tabMapping) { + private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind) { case INSERT: @@ -100,12 +100,10 @@ private void parseSql(SqlNode sqlNode, Map sideTa parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; - default: - break; } } - private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { + private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { predicateInfoList.stream().filter(info -> sideTableMap.containsKey(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable()))) .map(info -> sideTableMap.get(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable())).getPredicateInfoes().add(info)) .count(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index 2296f64c5..f37c3f78c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -47,8 +47,7 @@ import java.util.Queue; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.IDENTIFIER; -import static org.apache.calcite.sql.SqlKind.LITERAL; +import static org.apache.calcite.sql.SqlKind.*; /** * Parsing sql, obtain execution information dimension table @@ -63,6 +62,8 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { + System.out.println("----------exec original Sql----------"); + System.out.println(exeSql); LOG.info("----------exec original Sql----------"); LOG.info(exeSql); @@ -167,8 +168,6 @@ public Object parseSql(SqlNode sqlNode, case LITERAL: return LITERAL.toString(); - default: - break; } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index c41e2264a..f90138b2a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -43,17 +43,24 @@ import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import com.google.common.collect.*; +import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -61,6 +68,7 @@ import java.sql.Timestamp; import java.util.Arrays; +import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -89,7 +97,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); public void exec(String sql, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, @@ -123,9 +131,11 @@ public void exec(String sql, if(pollSqlNode.getKind() == INSERT){ + System.out.println("----------real exec sql-----------" ); + System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); if(LOG.isInfoEnabled()){ - LOG.info("----------real exec sql-----------\n{}", pollSqlNode.toString()); + LOG.info("exec sql: " + pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ @@ -156,7 +166,8 @@ public void exec(String sql, } }else if (pollObj instanceof JoinInfo){ - LOG.info("----------exec join info----------\n{}", pollObj.toString()); + System.out.println("----------exec join info----------"); + System.out.println(pollObj.toString()); joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -283,7 +294,7 @@ private Table getTableFromCache(Map localTableCache, String table * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -291,7 +302,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { + private List convertPrimaryAlias(SideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -299,7 +310,7 @@ private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -358,7 +369,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -374,7 +385,7 @@ private void joinFun(Object pollObj, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } @@ -383,9 +394,9 @@ private void joinFun(Object pollObj, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } - if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ - throw new RuntimeException("ON condition must contain all equal fields!!!"); - } +// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ +// throw new RuntimeException("ON condition must contain all equal fields!!!"); +// } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 7a832d0a1..8c78578fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ * @author xuchao */ -public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { +public abstract class SideTableInfo extends TableInfo implements Serializable { public static final String TARGET_SUFFIX = "Side"; @@ -55,8 +55,6 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; - public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; - private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -67,11 +65,6 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private int asyncTimeout=10000; - /** - * async operator req outside conn pool size, egg rdb conn pool size - */ - private int asyncPoolSize = 0; - private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; @@ -163,26 +156,4 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } - public int getAsyncPoolSize() { - return asyncPoolSize; - } - - public void setAsyncPoolSize(int asyncPoolSize) { - this.asyncPoolSize = asyncPoolSize; - } - - @Override - public String toString() { - return "Cache Info{" + - "cacheType='" + cacheType + '\'' + - ", cacheSize=" + cacheSize + - ", cacheTimeout=" + cacheTimeout + - ", asyncCapacity=" + asyncCapacity + - ", asyncTimeout=" + asyncTimeout + - ", asyncPoolSize=" + asyncPoolSize + - ", asyncTimeoutNumLimit=" + asyncTimeoutNumLimit + - ", partitionedJoin=" + partitionedJoin + - ", cacheMode='" + cacheMode + '\'' + - '}'; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 583e4597b..8417c4519 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.util.PluginUtil; /** @@ -37,7 +37,7 @@ public class StreamSideFactory { private static final String CURR_TYPE = "side"; - public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { + public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(cacheType) ? "all" : "async"; String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); @@ -45,10 +45,10 @@ public static AbstractTableParser getSqlParser(String pluginType, String sqlRoot return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sideParser = cl.loadClass(className); - if (!AbstractSideTableParser.class.isAssignableFrom(sideParser)) { + if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); } - return sideParser.asSubclass(AbstractTableParser.class).newInstance(); + return sideParser.asSubclass(AbsTableParser.class).newInstance(); }); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java similarity index 84% rename from core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java rename to core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java index 58832c583..757f91600 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; /** * Reason: @@ -30,11 +30,11 @@ * @author xuchao */ -public abstract class AbstractSideCache { +public abstract class AbsSideCache { - protected AbstractSideTableInfo sideTableInfo; + protected SideTableInfo sideTableInfo; - public AbstractSideCache(AbstractSideTableInfo sideTableInfo){ + public AbsSideCache(SideTableInfo sideTableInfo){ this.sideTableInfo = sideTableInfo; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 2664efd34..700e13bb2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,11 +33,11 @@ * @author xuchao */ -public class LRUSideCache extends AbstractSideCache { +public class LRUSideCache extends AbsSideCache{ protected transient Cache cache; - public LRUSideCache(AbstractSideTableInfo sideTableInfo) { + public LRUSideCache(SideTableInfo sideTableInfo) { super(sideTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 3c2010907..290804200 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; @@ -49,20 +49,20 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(BaseAsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) + cl.loadClass(className).asSubclass(AsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { - BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 6b6f9fe1b..5aa810b0f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; @@ -44,22 +44,22 @@ public class SideWithAllCacheOperator { private static final String OPERATOR_TYPE = "All"; - private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, - AbstractSideTableInfo sideTableInfo) throws Exception { + private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, + SideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(BaseAllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) + return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { - BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + AllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); return inputStream.flatMap(allReqRow); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java index e29421369..3cfd48f1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; /** * Reason: @@ -30,5 +30,5 @@ */ public interface IStreamSinkGener { - T genStreamSink(AbstractTargetTableInfo targetTableInfo); + T genStreamSink(TargetTableInfo targetTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 89061db3a..53460081d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.table.sinks.TableSink; @@ -40,21 +40,21 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; - public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class targetParser = cl.loadClass(className); - if(!AbstractTableParser.class.isAssignableFrom(targetParser)){ + if(!AbsTableParser.class.isAssignableFrom(targetParser)){ throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); } - return targetParser.asSubclass(AbstractTableParser.class).newInstance(); + return targetParser.asSubclass(AbsTableParser.class).newInstance(); }); } - public static TableSink getTableSink(AbstractTargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); diff --git a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java index fde5a235d..b8b8f6edc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -38,6 +38,6 @@ public interface IStreamSourceGener { * @param tableEnv * @return */ - T genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); + T genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index e0cec1415..1057fb0ed 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbstractSourceParser; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -43,17 +43,17 @@ public class StreamSourceFactory { private static final String DIR_NAME_FORMAT = "%ssource"; - public static AbstractSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sourceParser = cl.loadClass(className); - if(!AbstractSourceParser.class.isAssignableFrom(sourceParser)){ + if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); } - return sourceParser.asSubclass(AbstractSourceParser.class).newInstance(); + return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); }); } @@ -62,7 +62,7 @@ public static AbstractSourceParser getSqlParser(String pluginType, String sqlRoo * @param sourceTableInfo * @return */ - public static Table getStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, + public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String sqlRootDir) throws Exception { String sourceTypeStr = sourceTableInfo.getType(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java similarity index 62% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 4973a891e..bf0b8a7a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -21,10 +21,8 @@ package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.flink.util.Preconditions; - import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -36,24 +34,24 @@ * @author xuchao */ -public abstract class AbstractSideTableParser extends AbstractTableParser { +public abstract class AbsSideTableParser extends AbsTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; private final static Pattern SIDE_TABLE_SIGN = Pattern.compile("(?i)^PERIOD\\s+FOR\\s+SYSTEM_TIME$"); - public AbstractSideTableParser() { + public AbsSideTableParser() { addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } - private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo){ + private void dealSideSign(Matcher matcher, TableInfo tableInfo){ //FIXME SIDE_TABLE_SIGN current just used as a sign for side table; and do nothing } //Analytical create table attributes ==> Get information cache - protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map props){ - if(props.containsKey(AbstractSideTableInfo.CACHE_KEY.toLowerCase())){ - String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY.toLowerCase())); + protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ + if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ + String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); if(cacheType == null){ return; } @@ -63,66 +61,59 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ - Long cacheTTLMS = MathUtil.getLongVal(props.get(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); if(cacheTTLMS < 1000){ throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ - Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); if(partitionedJoinKey){ sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ + if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); if(asyncCap < 0){ throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); if (asyncTimeout<0){ throw new RuntimeException("asyncTimeout size need > 0."); } sideTableInfo.setAsyncTimeout(asyncTimeout); } - if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ - Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); + if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ + Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } - - if (props.containsKey(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { - Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); - Preconditions.checkArgument(asyncPoolSize > 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); - sideTableInfo.setAsyncPoolSize(asyncPoolSize); - } - } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java similarity index 81% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java index 308f5859f..745357162 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java @@ -33,7 +33,7 @@ * @author xuchao */ -public abstract class AbstractSourceParser extends AbstractTableParser { +public abstract class AbsSourceParser extends AbsTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; private static final String WATERMARK_KEY = "waterMarkKey"; @@ -43,21 +43,21 @@ public abstract class AbstractSourceParser extends AbstractTableParser { private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); - public AbstractSourceParser() { + public AbsSourceParser() { addParserHandler(VIRTUAL_KEY, virtualFieldKeyPattern, this::dealVirtualField); addParserHandler(WATERMARK_KEY, waterMarkKeyPattern, this::dealWaterMark); addParserHandler(NOTNULL_KEY, notNullKeyPattern, this::dealNotNull); } - protected void dealVirtualField(Matcher matcher, AbstractTableInfo tableInfo){ - AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; + protected void dealVirtualField(Matcher matcher, TableInfo tableInfo){ + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; String fieldName = matcher.group(2); String expression = matcher.group(1); sourceTableInfo.addVirtualField(fieldName, expression); } - protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ - AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; + protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ + SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; String eventTimeField = matcher.group(1); //FIXME Temporarily resolve the second parameter row_time_field Integer offset = MathUtil.getIntegerVal(matcher.group(3)); @@ -65,11 +65,11 @@ protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ sourceTableInfo.setMaxOutOrderness(offset); } - protected void dealNotNull(Matcher matcher, AbstractTableInfo tableInfo) { + protected void dealNotNull(Matcher matcher, TableInfo tableInfo) { String fieldName = matcher.group(1); String fieldType = matcher.group(2); Class fieldClass= dbTypeConvertToJavaType(fieldType); - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(true); tableInfo.addPhysicalMappings(fieldName, fieldName); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java similarity index 88% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index 1f210cbf2..3e4027b2c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -39,7 +39,7 @@ * @author xuchao */ -public abstract class AbstractTableParser { +public abstract class AbsTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; @@ -54,7 +54,7 @@ public abstract class AbstractTableParser { private Map handlerMap = Maps.newHashMap(); - public AbstractTableParser() { + public AbsTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } @@ -63,9 +63,9 @@ protected boolean fieldNameNeedsUpperCase() { return true; } - public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; + public abstract TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ + public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ for(Map.Entry keyPattern : patternMap.entrySet()){ Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); @@ -84,7 +84,7 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ @@ -112,12 +112,12 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ Class fieldClass = null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + TableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); @@ -133,7 +133,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); @@ -145,7 +145,7 @@ public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ * @param matcher * @param tableInfo */ - protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { + protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String physicalField = matcher.group(1); Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); @@ -154,7 +154,7 @@ protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { String mappingField = matcher.group(4); Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java index fab634cdc..db804ea34 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java @@ -30,5 +30,5 @@ */ public interface ITableFieldDealHandler { - void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo); + void dealPrimaryKey(Matcher matcher, TableInfo tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java index 0fdc0e911..9a41fa0a1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java @@ -36,7 +36,7 @@ * @author xuchao */ -public abstract class AbstractSourceTableInfo extends AbstractTableInfo { +public abstract class SourceTableInfo extends TableInfo { public static final String SOURCE_SUFFIX = "Source"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java similarity index 98% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java index 3b514dd48..2fdc297a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java @@ -34,7 +34,7 @@ * @author xuchao */ -public abstract class AbstractTableInfo implements Serializable { +public abstract class TableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java similarity index 83% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index d50e1f23c..3c1be25c6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.StreamSideFactory; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; @@ -42,7 +42,7 @@ * @author xuchao */ -public class AbstractTableInfoParser { +public class TableInfoParser { private final static String TYPE_KEY = "type"; @@ -50,16 +50,16 @@ public class AbstractTableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); //Parsing loaded plugin - public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot) throws Exception { - AbstractTableParser absTableParser = null; + public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, + String localPluginRoot) throws Exception { + AbsTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); @@ -77,9 +77,9 @@ public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.Sql sourceTableInfoMap.put(type, absTableParser); } }else{ - absTableParser = sideTableInfoMap.get(type); + String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); + absTableParser = sideTableInfoMap.get(type + cacheType); if(absTableParser == null){ - String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); sideTableInfoMap.put(type + cacheType, absTableParser); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java index dbd99f993..0dc9ca77b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java @@ -29,7 +29,7 @@ * @author xuchao */ -public abstract class AbstractTargetTableInfo extends AbstractTableInfo { +public abstract class TargetTableInfo extends TableInfo { public static final String TARGET_SUFFIX = "Sink"; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 4bb4ff0d8..feebbe3f1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -86,8 +86,7 @@ public static Class stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - default: - break; + } throw new RuntimeException("不支持 " + str + " 类型"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index 409602baa..a0f5291e7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -16,17 +16,22 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.*; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.*; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; +import java.util.SimpleTimeZone; +import java.util.TimeZone; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; @@ -34,7 +39,7 @@ /** * - * 日期工具 + * Reason: TODO ADD REASON(可选) * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com * @author sishu.yss @@ -42,14 +47,18 @@ */ public class DateUtil { - static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); + static final String timeZone = "GMT+8"; + static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; + static final String dateFormat = "yyyy-MM-dd"; + static final String timeFormat = "HH:mm:ss"; + static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); + static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); + static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final int MILLIS_PER_SECOND = 1000; + private static final int MILLIS_PER_SECOND = 1000; public static java.sql.Date columnToDate(Object column) { @@ -75,32 +84,23 @@ public static Date stringToDate(String strDate) { return null; } try { - ; - return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); - } catch (DateTimeParseException ignored) { + return datetimeFormatter.parse(strDate); + } catch (ParseException ignored) { } try { - return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); - } catch (DateTimeParseException ignored) { + return dateFormatter.parse(strDate); + } catch (ParseException ignored) { } try { - return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); - } catch (DateTimeParseException ignored) { + return timeFormatter.parse(strDate); + } catch (ParseException ignored) { } throw new RuntimeException("can't parse date"); } - public static Date localDateTimetoDate(LocalDateTime localDateTime){ - return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); - } - - public static LocalDateTime dateToLocalDateTime(Date date){ - return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); - } - /** * * @@ -130,13 +130,13 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if("MS".equals(scope)){ - return getTodayStart(day)*1000; - }else if("S".equals(scope)){ - return getTodayStart(day); - }else{ - return getTodayStart(day); - } + if(scope.equals("MS")){ + return getTodayStart(day)*1000; + }else if(scope.equals("S")){ + return getTodayStart(day); + }else{ + return getTodayStart(day); + } } /** @@ -168,15 +168,15 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if("MS".equals(scope)){ - return getNextDayStart(day)*1000; - }else if("S".equals(scope)){ - return getNextDayStart(day); - }else{ - return getNextDayStart(day); - } + if(scope.equals("MS")){ + return getNextDayStart(day)*1000; + }else if(scope.equals("S")){ + return getNextDayStart(day); + }else{ + return getNextDayStart(day); + } } - + /** * @@ -237,7 +237,7 @@ public static long getWeekFirst(long day) { /** * 根据某个日期时间戳秒值，获取所在周在一年中是第几周. - * + * * @param day * @return */ @@ -257,17 +257,17 @@ public static int getWeekOfYear(long day) { */ public static String getYesterdayByString(String day, String inFormat, String outFormat){ try { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay - 1); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; - } catch (ParseException e) { - return null; - } + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay - 1); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; + } catch (ParseException e) { + return null; + } } /** @@ -288,7 +288,7 @@ public static String getTomorrowByString(String day, String inFormat, String out String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - + /** * * @param date @@ -320,7 +320,7 @@ public static String get30DaysBeforeByString(String day, String inFormat, String calendar.set(Calendar.DATE, calendarDay - 30); return new SimpleDateFormat(outFormat).format(calendar.getTime()); } - + /** * * @param day @@ -330,14 +330,14 @@ public static String get30DaysBeforeByString(String day, String inFormat, String * @throws ParseException */ public static String get30DaysLaterByString(String day, String inFormat, String outFormat) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay + 30); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay + 30); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; } @@ -349,7 +349,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -357,8 +357,8 @@ public static String getDateStrToFormat(String day, String inFormat, String outF String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - - public static long getDateMillToFormat(String day, String inFormat) throws ParseException { + + public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -474,79 +474,79 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MILLISECOND, 0); return cal.getTimeInMillis() / 1000; } - - /** - * @return long - */ + + /** + * @return long + */ public static long getMillByDay(int severalDays,String condition) { - int dateT=0; + int dateT=0; Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getMillToDay(cal,dateT); - } - if("-".equals(condition)){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getMillToDay(cal,dateT); - } - if("+".equals(condition)){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getMillToDay(cal,dateT); - } - return getMillToDay(cal,dateT); - } - + if(condition==null){ + return getMillToDay(cal,dateT); + } + if(condition.equals("-")){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getMillToDay(cal,dateT); + } + if(condition.equals("+")){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getMillToDay(cal,dateT); + } + return getMillToDay(cal,dateT); + } + /** * @return long */ public static long getStampByDay(int severalDays,String condition) { - int dateT=0; - Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getStampToDay(cal,dateT); - } - if("-".equals(condition)){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getStampToDay(cal,dateT); - } - if("+".equals(condition)){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getStampToDay(cal,dateT); - } - return getStampToDay(cal,dateT); + int dateT=0; + Calendar cal = Calendar.getInstance(); + if(condition==null){ + return getStampToDay(cal,dateT); + } + if(condition.equals("-")){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getStampToDay(cal,dateT); + } + if(condition.equals("+")){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getStampToDay(cal,dateT); + } + return getStampToDay(cal,dateT); } /** * @return long */ public static long getMillByDay(){ - return getMillByDay(0,null); + return getMillByDay(0,null); } - + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getMillToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; - } - + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis()/1000; + } + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getStampToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - return cal.getTimeInMillis(); + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + return cal.getTimeInMillis(); } public static String getToday() { @@ -568,7 +568,7 @@ public static String getDate(long day, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); } - + /** * * @param date @@ -578,21 +578,21 @@ public static String getDate(Date date, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(date); } - - + + /** * * @param day * @param format * @return long - * @throws ParseException + * @throws ParseException */ public static long stringToLong(String day, String format) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long date = dateFormat.parse(day).getTime(); - return date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + long Date = dateFormat.parse(day).getTime(); + return Date; } - + /** * @param day * @param format @@ -600,30 +600,30 @@ public static long stringToLong(String day, String format) throws ParseException * @throws ParseException */ public static Date stringToDate(String day, String format) { - try { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date date = dateFormat.parse(day); - return date; - } catch (ParseException e) { - return new Date(); - } + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + Date Date = dateFormat.parse(day); + return Date; + } catch (ParseException e) { + return new Date(); + } } - - + + /** * long型时间戳转为String型 - * + * * @param day 秒 * @return 格式化后的日期 - * @throws ParseException + * @throws ParseException */ public static String longToString(long day, String format) throws ParseException { - if (("" + day).length() <= 10){ + if (("" + day).length() <= 10){ day=day*1000; } - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String date = dateFormat.format(day); - return date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + String Date = dateFormat.format(day); + return Date; } /** @@ -653,39 +653,39 @@ public static long getMillByNow() { return cal.getTimeInMillis(); } - public static int getWeeksBetweenTwoDates(long startDay, long endDay) { - int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; - if(week<1){ - week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; - } - return week; - } + public static int getWeeksBetweenTwoDates(long startDay, long endDay) { + int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; + if(week<1){ + week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; + } + return week; + } - public static int getMaxWeekOfYear(long startDay) { - Calendar cal = Calendar.getInstance(); + public static int getMaxWeekOfYear(long startDay) { + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(startDay * 1000)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); - } - - public static int getMonthsBetweenTwoDates(long startDay, long endDay) { - int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; - if(month<1){ - month = getMonth(endDay) + 12 - getMonth(startDay) +1; - } - return month; - } - - public static Date parseDate(String dateStr, String pattern){ - SimpleDateFormat sdf = new SimpleDateFormat(); - sdf.applyPattern(pattern); - try { - return sdf.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - - /** + } + + public static int getMonthsBetweenTwoDates(long startDay, long endDay) { + int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; + if(month<1){ + month = getMonth(endDay) + 12 - getMonth(startDay) +1; + } + return month; + } + + public static Date parseDate(String dateStr, String pattern){ + SimpleDateFormat sdf = new SimpleDateFormat(); + sdf.applyPattern(pattern); + try { + return sdf.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + + /** * * @param time Long 时间 * @return long @@ -703,7 +703,7 @@ public static long getMinuteStart(long time) { firstDay = cal.getTimeInMillis() / 1000; return firstDay; } - + /** * @param time Long * @return long @@ -732,24 +732,24 @@ public static Date getDateByLong(long time){ date.setTime(time); return date; } - + public static Date parseDate(String dateStr, String pattern, Locale locale){ - SimpleDateFormat df = new SimpleDateFormat( - pattern, locale); - - df.setTimeZone(new SimpleTimeZone(0, "GMT")); - try { - return df.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - + SimpleDateFormat df = new SimpleDateFormat( + pattern, locale); + + df.setTimeZone(new SimpleTimeZone(0, "GMT")); + try { + return df.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + public static String getDate(Date date, String format, Locale locale) { - SimpleDateFormat df = new SimpleDateFormat( - format, locale); - df.setTimeZone(new SimpleTimeZone(0, "GMT")); + SimpleDateFormat df = new SimpleDateFormat( + format, locale); + df.setTimeZone(new SimpleTimeZone(0, "GMT")); return df.format(date); } @@ -777,33 +777,59 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - LocalDateTime localDateTime = dateToLocalDateTime(date); - return localDateTime.format(DATE_FORMATTER); + return dateFormatter.format(date); } public static String timestampToString(Date date) { - LocalDateTime localDateTime = dateToLocalDateTime(date); - return localDateTime.format(DATE_TIME_FORMATTER); + return datetimeFormatter.format(date); } + public static Timestamp getTimestampFromStr(String timeStr) { if (DATETIME.matcher(timeStr).matches()) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); + } else { + java.sql.Date date = null; + try { + date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("getTimestampFromStr error data is " + timeStr); + } + return new Timestamp(date.getTime()); } - return new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { + // 2020-01-01 format if (DATE.matcher(dateStr).matches()) { + // convert from local date to instant Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); + // calculate the timezone offset in millis int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); + // need to remove the offset since time has no TZ component return new java.sql.Date(instant.toEpochMilli() - offset); } else if (DATETIME.matcher(dateStr).matches()) { + // 2020-01-01T12:12:12Z format Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); + } else { + try { + // 2020-01-01 12:12:12.0 format + return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); + } catch (ParseException e) { + throw new RuntimeException("String convert to Date fail."); + } } - return new java.sql.Date(stringToDate(dateStr).getTime()); + } + + + public static String getStringFromTimestamp(Timestamp timestamp) { + return datetimeFormatter.format(timestamp); + } + + public static String getStringFromDate(java.sql.Date date) { + return dateFormatter.format(date); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 10a6b4f63..b2486b1ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -219,14 +219,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return preStr + "?" + sb.toString(); } - public static boolean isJson(String str) { + public static boolean isJosn(String str){ boolean flag = false; - if (StringUtils.isNotBlank(str)) { + if(StringUtils.isNotBlank(str)){ try { - objectMapper.readValue(str, Map.class); + objectMapper.readValue(str,Map.class); flag = true; } catch (Throwable e) { - flag = false; + flag=false; } } return flag; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index 3d66d8a6d..fde2f166e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -28,10 +28,10 @@ public class JDBCUtils { private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - public final static String LOCK_STR = "jdbc_lock_str"; + public final static String lock_str = "jdbc_lock_str"; public static void forName(String clazz, ClassLoader classLoader) { - synchronized (LOCK_STR){ + synchronized (lock_str){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 5aaa40a15..23513a912 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -227,7 +227,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return BigDecimal.valueOf(((Number) obj).doubleValue()); + return new BigDecimal(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index 8eb743cf0..a63e5a663 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -109,7 +109,7 @@ public static String getSqlSideClassName(String pluginTypeName, String type, Str return CLASS_PRE_STR + "." + type.toLowerCase() + "." + pluginTypeName + "." + pluginClassName; } - public static Map objectToMap(Object obj) throws Exception{ + public static Map ObjectToMap(Object obj) throws Exception{ return objectMapper.readValue(objectMapper.writeValueAsBytes(obj), Map.class); } @@ -173,8 +173,8 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } for(File file : files){ - URL pluginJarUrl = file.toURI().toURL(); - classLoader.addURL(pluginJarUrl); + URL pluginJarURL = file.toURI().toURL(); + classLoader.addURL(pluginJarURL); } } @@ -191,8 +191,8 @@ public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLExcept } for(File file : files){ - URL pluginJarUrl = file.toURI().toURL(); - urlList.add(pluginJarUrl); + URL pluginJarURL = file.toURI().toURL(); + urlList.add(pluginJarURL); } return urlList.toArray(new URL[urlList.size()]); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java rename to core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java index d75d26a61..ed8f64288 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java @@ -38,7 +38,7 @@ * @author xuchao */ -public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { +public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { private static final long serialVersionUID = 1L; @@ -55,7 +55,7 @@ public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrderne protected TimeZone timezone; - public AbstractCustomerWaterMarker(Time maxOutOfOrderness) { + public AbsCustomerWaterMarker(Time maxOutOfOrderness) { super(maxOutOfOrderness); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java index 979b4c3d6..99415ba61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java @@ -35,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForLong extends AbstractCustomerWaterMarker { +public class CustomerWaterMarkerForLong extends AbsCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForLong.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java index 05ee3e46e..b7961e410 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.watermarker; +import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -35,7 +36,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForTimeStamp extends AbstractCustomerWaterMarker { +public class CustomerWaterMarkerForTimeStamp extends AbsCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForTimeStamp.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index b0ca0335f..f3208db03 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; @@ -40,7 +40,7 @@ public class WaterMarkerAssigner { - public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ + public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ return false; } @@ -48,7 +48,7 @@ public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ return true; } - public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ + public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, SourceTableInfo sourceTableInfo){ String eventTimeFieldName = sourceTableInfo.getEventTimeField(); @@ -75,7 +75,7 @@ public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo type TypeInformation fieldType = fieldTypes[pos]; - AbstractCustomerWaterMarker waterMarker = null; + AbsCustomerWaterMarker waterMarker = null; if(fieldType.getTypeClass().isAssignableFrom(Timestamp.class)){ waterMarker = new CustomerWaterMarkerForTimeStamp(Time.milliseconds(maxOutOrderness), pos,timeZone); }else if(fieldType.getTypeClass().isAssignableFrom(Long.class)){ diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java index feec8e547..546fcbbac 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java @@ -54,7 +54,7 @@ public void testfillPredicatesForSideTable() throws SqlParseException { " MyTable.a='1' and s.d='1' and s.d <> '3' and s.c LIKE '%xx%' and s.c in ('1','2') and s.c between '10' and '23' and s.d is not null\n"; - AbstractSideTableInfo sideTableInfo = new AbstractSideTableInfo(){ + SideTableInfo sideTableInfo = new SideTableInfo(){ @Override public boolean check() { return false; @@ -63,7 +63,7 @@ public boolean check() { sideTableInfo.setName("sideTable"); - Map sideTableMap = new HashMap<>(); + Map sideTableMap = new HashMap<>(); sideTableMap.put("sideTable", sideTableInfo); SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index b354a6269..a77201b80 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -20,8 +20,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +31,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; +import java.util.Map; /** * Date: 2019/11/20 @@ -37,21 +40,21 @@ * @author xiuzhu */ -public class Db2AllReqRow extends AbstractRdbAllReqRow { +public class Db2AllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(Db2AllReqRow.class); private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Db2AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(DB2_DRIVER); - Connection conn = DriverManager.getConnection(dbUrl, userName, password); + Connection conn = DriverManager.getConnection(dbURL, userName, password); return conn; } catch (Exception e) { LOG.error("", e); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index 0120c58a0..282ee440c 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AllSideInfo extends RdbAllSideInfo { - public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 8e7275977..3d85c6fbd 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class Db2AsyncReqRow extends RdbAsyncReqRow { private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Db2AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); db2lientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", DB2_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -71,10 +71,10 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index 9cbdc997e..e53dd7f27 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { - public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index 96be4ec15..db8a07dc6 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.db2.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class Db2SideParser extends RdbSideParser { private static final String CURR_TYPE = "db2"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index da1434ace..9942d4438 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,10 +1,13 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -public class DbSink extends AbstractRdbSink { +import java.util.List; +import java.util.Map; + +public class DbSink extends RdbSink { public DbSink() { super(new DbDialect()); @@ -12,7 +15,7 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index 3e823ab20..ba11aabf3 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.sink.db.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -10,8 +10,8 @@ public class DbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "db2"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; } diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md deleted file mode 100644 index 5d7c7b2a7..000000000 --- a/docs/kafkaSink.md +++ /dev/null @@ -1,223 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - function(colNameX) AS aliasName, - WATERMARK FOR colName AS withOffset( colName , delayTime ) - )WITH( - type ='kafka11', - bootstrapServers ='ip:port,ip:port...', - zookeeperQuorum ='ip:port,ip:port/zkparent', - offsetReset ='latest', - topic ='topicName', - groupId='test', - parallelism ='parllNum', - ); -``` - -## 2.支持的版本 - kafka09,kafka10,kafka11及以上版本 - **kafka读取和写入的版本必须一致，否则会有兼容性错误。** - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| -|groupId | 需要读取的 groupId 名称|否|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|parallelism | 并行度设置|否|1| -|partitionKeys | 用来分区的字段|否|| -|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| -|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| -|fieldDelimiter | csv数据分隔符|否| , | - - -**kafka相关参数可以自定义，使用kafka.开头即可。** -``` -kafka.consumer.id -kafka.socket.timeout.ms -kafka.fetch.message.max.bytes -kafka.num.consumer.fetchers -kafka.auto.commit.enable -kafka.auto.commit.interval.ms -kafka.queued.max.message.chunks -kafka.rebalance.max.retries -kafka.fetch.min.bytes -kafka.fetch.wait.max.ms -kafka.rebalance.backoff.ms -kafka.refresh.leader.backoff.ms -kafka.consumer.timeout.ms -kafka.exclude.internal.topics -kafka.partition.assignment.strategy -kafka.client.id -kafka.zookeeper.session.timeout.ms -kafka.zookeeper.connection.timeout.ms -kafka.zookeeper.sync.time.ms -kafka.offsets.storage -kafka.offsets.channel.backoff.ms -kafka.offsets.channel.socket.timeout.ms -kafka.offsets.commit.max.retries -kafka.dual.commit.enabled -kafka.partition.assignment.strategy -kafka.socket.receive.buffer.bytes -kafka.fetch.min.bytes -``` - -## 5.样例： - -### json格式： -``` -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - partitionKeys = 'channel,pv', - updateMode='upsert' - ); - -upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} -append模式下发的数据格式：{"channel":"zs","pv":"330"} - -``` - -### avro格式： - -如果updateMode='upsert',schemaInfo需要包含retract属性信息。 - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='1', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - --type='console' - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='1', - updateMode='upsert', - sinkdatatype = 'avro', - schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} - ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, - {"name":"retract","type":"boolean"}]}' - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` -### csv格式： - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='2', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - updateMode='upsert', - sinkdatatype = 'csv', - fieldDelimiter='*' - - - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 00c252c99..76096aba6 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -1,5 +1,6 @@ ## 1.格式： ``` +数据现在支持json格式{"xx":"bb","cc":"dd"} CREATE TABLE tableName( colName colType, @@ -14,8 +15,9 @@ CREATE TABLE tableName( topic ='topicName', groupId='test', parallelism ='parllNum', + --timezone='America/Los_Angeles', timezone='Asia/Shanghai', - sourcedatatype ='dt_nest' #可不设置 + sourcedatatype ='json' #可不设置 ); ``` @@ -45,9 +47,7 @@ CREATE TABLE tableName( |topicIsPattern | topic是否是正则表达式格式(true|false) |否| false |offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| -|schemaInfo | avro类型使用的schema信息|否|| -|fieldDelimiter |csv类型使用的数据分隔符|否| | | +|sourcedatatype | 数据类型|否|json| |timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` @@ -169,10 +169,24 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` +# 二、csv格式数据源 +根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 +## 1.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| +|parallelism | 并行度设置 |否|1| +|sourcedatatype | 数据类型|是 |csv| +|fielddelimiter | 字段分隔符|是 || +|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| +**kafka相关参数可以自定义，使用kafka.开头即可。** -## 7.csv格式数据源 - - +## 2.样例： ``` CREATE TABLE MyTable( name varchar, @@ -189,28 +203,186 @@ CREATE TABLE MyTable( --topic ='mqTest.*', --topicIsPattern='true' parallelism ='1', - sourceDatatype ='csv' + sourcedatatype ='csv', + fielddelimiter ='\|', + lengthcheckpolicy = 'PAD' ); ``` -## 8.avro格式数据源 +# 三、text格式数据源UDF自定义拆分 +Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 + 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： + +## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 +``` +create table kafka_stream( + _topic STRING, + _messageKey STRING, + _message STRING, + _partition INT, + _offset BIGINT, +) with ( + type ='kafka09', + bootstrapServers ='172.16.8.198:9092', + zookeeperQuorum ='172.16.8.198:2181/kafka', + offsetReset ='latest', + topic ='nbTest1', + parallelism ='1', + sourcedatatype='text' + ） +``` +## 2.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| +|parallelism | 并行度设置|否|1| +|sourcedatatype | 数据类型|否|text| +**kafka相关参数可以自定义，使用kafka.开头即可。** +## 2.自定义： +从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， +需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： +2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） + +**SQL** ``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='1', - topicIsPattern ='false', - kafka.group.id='mqTest', - sourceDataType ='avro', - schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"},{"name":"pv","type":"string"}]}' +-- 定义解析Kakfa message的UDTF + CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; + CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; + -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 + create table kafka_src ( + _topic STRING, + _messageKey STRING, + _message STRING, + _partition INT, + _offset BIGINT, + ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 + watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark + ) WITH ( + type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 + topic = 'test_kafka_topic', + ... ); - + create table rds_sink ( + name VARCHAR, + age INT, + grade VARCHAR, + updateTime TIMESTAMP + ) WITH( + type='mysql', + url='jdbc:mysql://localhost:3306/test', + tableName='test4', + userName='test', + password='XXXXXX' + ); + -- 使用UDTF，将二进制数据解析成格式化数据 + CREATE VIEW input_view ( + name, + age, + grade, + updateTime + ) AS + SELECT + COUNT(*) as cnt, + T.ctime, + T.order, + T.name, + T.sex + from + kafka_src as S, + LATERAL TABLE (kafkapaser _message)) as T ( + ctime, + order, + name, + sex + ) + Group BY T.sex, + TUMBLE(ctime, INTERVAL '1' MINUTE); + -- 对input_view中输出的数据做计算 + CREATE VIEW view2 ( + cnt, + sex + ) AS + SELECT + COUNT(*) as cnt, + T.sex + from + input_view + Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); + -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 + insert into rds_sink + SELECT + cnt,sex + from view2; + ``` +**UDF&UDTF** ``` - +package com.XXXX; + import com.XXXX.fastjson.JSONObject; + import org.apache.flink.table.functions.TableFunction; + import org.apache.flink.table.types.DataType; + import org.apache.flink.table.types.DataTypes; + import org.apache.flink.types.Row; + import java.io.UnsupportedEncodingException; + /** + 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 + **/ + public class kafkaUDTF extends TableFunction { + public void eval(byte[] message) { + try { + // 读入一个二进制数据，并将其转换为String格式 + String msg = new String(message, "UTF-8"); + // 提取JSON Object中各字段 + String ctime = Timestamp.valueOf(data.split('\\|')[0]); + String order = data.split('\\|')[1]; + String name = data.split('\\|')[2]; + String sex = data.split('\\|')[3]; + // 将解析出的字段放到要输出的Row()对象 + Row row = new Row(4); + row.setField(0, ctime); + row.setField(1, age); + row.setField(2, grade); + row.setField(3, updateTime); + System.out.println("Kafka message str ==>" + row.toString()); + // 输出一行 + collect(row); + } catch (ClassCastException e) { + System.out.println("Input data format error. Input data " + msg + "is not json string"); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + @Override + // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 + // 定义输出Row()对象的字段类型 + public DataType getResultType(Object[] arguments, Class[] argTypes) { + return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); + } + } + + package com.dp58; + package com.dp58.sql.udx; + import org.apache.flink.table.functions.FunctionContext; + import org.apache.flink.table.functions.ScalarFunction; + public class KafkaUDF extends ScalarFunction { + // 可选，open方法可以不写 + // 需要import org.apache.flink.table.functions.FunctionContext; + public String eval(byte[] message) { + // 读入一个二进制数据，并将其转换为String格式 + String msg = new String(message, "UTF-8"); + return msg.split('\\|')[0]; + } + public long eval(String b, String c) { + return eval(b) + eval(c); + } + //可选，close方法可以不写 + @Override + public void close() { + } + } + ``` diff --git a/docs/mongoSide.md b/docs/mongoSide.md index 73ce9644f..db557f441 100644 --- a/docs/mongoSide.md +++ b/docs/mongoSide.md @@ -40,6 +40,8 @@ |----|---|---|----| | type |表明 输出表类型 mongo|是|| | address | 连接mongo数据库 jdbcUrl |是|| + | userName | mongo连接用户名|否|| + | password | mongo连接密码|否|| | tableName | mongo表名称|是|| | database | mongo表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| @@ -62,8 +64,7 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/docs/mongoSink.md b/docs/mongoSink.md index b5bb6df0d..b0f916aab 100644 --- a/docs/mongoSink.md +++ b/docs/mongoSink.md @@ -33,6 +33,8 @@ CREATE TABLE tableName( |----|----|----|----| |type |表明 输出表类型 mongo|是|| |address | 连接mongo数据库 jdbcUrl |是|| +|userName | mongo连接用户名|否|| +|password | mongo连接密码|否|| |tableName | mongo表名称|是|| |database | mongo表名称|是|| |parallelism | 并行度设置|否|1| @@ -44,8 +46,9 @@ CREATE TABLE MyResult( pv VARCHAR )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', + userName ='dtstack', + password ='abc123', database ='test', tableName ='pv', parallelism ='1' diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 5ca81c5ed..491d1af27 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.sink.elasticsearch; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -34,6 +33,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,7 +164,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 7988e597c..93682de46 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,7 +32,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchSinkParser extends AbstractTableParser { +public class ElasticsearchSinkParser extends AbsTableParser { private static final String KEY_ES_ADDRESS = "address"; @@ -56,7 +56,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index f254d5b4e..9681742c2 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -33,7 +33,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchTableInfo extends AbstractTargetTableInfo { +public class ElasticsearchTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "elasticsearch"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index b8cf2c46a..648f09fba 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -18,22 +18,24 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -47,6 +49,7 @@ import java.io.IOException; import java.io.Serializable; +import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; import java.util.List; @@ -57,7 +60,7 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializable { +public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); @@ -67,7 +70,7 @@ public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializab private SearchRequest searchRequest; private BoolQueryBuilder boolQueryBuilder; - public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index d353a583d..bf21b3ca8 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -18,13 +18,12 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; - +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; @@ -36,15 +35,15 @@ * @author yinxi * @date 2020/1/13 - 1:01 */ -public class Elasticsearch6AllSideInfo extends BaseSideInfo { +public class Elasticsearch6AllSideInfo extends SideInfo { - public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 1f4bd1bf1..f67177fe4 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,12 +18,6 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -32,6 +26,7 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; @@ -62,14 +57,14 @@ * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { +public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); SqlNode conditionNode = joinInfo.getCondition(); ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index 4118988ab..7b3a2f9e7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -18,17 +18,18 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -36,15 +37,15 @@ * @author yinxi * @date 2020/2/13 - 13:09 */ -public class Elasticsearch6AsyncSideInfo extends BaseSideInfo { +public class Elasticsearch6AsyncSideInfo extends SideInfo { - public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index 1b39bbf0f..13dfe2995 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -30,7 +30,7 @@ * @author yinxi * @date 2020/1/13 - 1:07 */ -public class Elasticsearch6SideParser extends AbstractSideTableParser { +public class Elasticsearch6SideParser extends AbsSideTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); elasticsearch6SideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java index a01c90a53..0afe2d59e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -26,7 +26,7 @@ * @author yinxi * @date 2020/1/13 - 15:00 */ -public class Elasticsearch6SideTableInfo extends AbstractSideTableInfo { +public class Elasticsearch6SideTableInfo extends SideTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index 3d9e4d62a..cc591789e 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; -import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; @@ -98,7 +98,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } // add index and type to search request - public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { + public static SearchRequest setSearchRequest(SideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); // determine existence of index @@ -129,7 +129,7 @@ public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { } // build where cause - public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { + public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -144,7 +144,7 @@ public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { } // build filter condition - public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, BaseSideInfo sideInfo) { + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); @@ -202,7 +202,7 @@ public static String[] removeSpaceAndApostrophe(String str) { } // prevent word segmentation - public static String textConvertToKeyword(String fieldName, BaseSideInfo sideInfo) { + public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); String fieldType = sideFieldTypes[fieldIndex]; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index b7d9de6fc..2646c50e9 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.sink.elasticsearch; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -32,6 +31,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -128,7 +128,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; clusterName = esTableInfo.getClusterName(); index = esTableInfo.getIndex(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 22c2b72bc..8902b953c 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchSinkParser extends AbstractTableParser { +public class ElasticsearchSinkParser extends AbsTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index d0eef18f0..3cc3dd9ff 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchTableInfo extends AbstractTargetTableInfo { +public class ElasticsearchTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index ed2931fff..11380eea6 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -45,7 +45,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; -public class HbaseAllReqRow extends BaseAllReqRow { +public class HbaseAllReqRow extends AllReqRow { private static final Logger LOG = LoggerFactory.getLogger(HbaseAllReqRow.class); @@ -55,7 +55,7 @@ public class HbaseAllReqRow extends BaseAllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); - public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -134,7 +134,7 @@ public void flatMap(CRow input, Collector out) throws Exception { Map cacheList = null; - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; if (hbaseSideTableInfo.isPreRowKey()) { for (Map.Entry> entry : cacheRef.get().entrySet()) { @@ -153,7 +153,7 @@ public void flatMap(CRow input, Collector out) throws Exception { } private void loadData(Map> tmpCache) throws SQLException { - AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); @@ -182,17 +182,9 @@ private void loadData(Map> tmpCache) throws SQLExcep LOG.error("", e); } finally { try { - if (null != conn && !conn.isClosed()) { - conn.close(); - } - - if (null != table) { - table.close(); - } - - if (null != resultScanner) { - resultScanner.close(); - } + conn.close(); + table.close(); + resultScanner.close(); } catch (IOException e) { LOG.error("", e); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index 663c2927b..ea51f46e4 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ import java.util.List; -public class HbaseAllSideInfo extends BaseSideInfo { +public class HbaseAllSideInfo extends SideInfo { private RowKeyBuilder rowKeyBuilder; - public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 24ee01786..251b88034 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -21,12 +21,12 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; +import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbsRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; @@ -58,7 +58,7 @@ * @author xuchao */ -public class HbaseAsyncReqRow extends BaseAsyncReqRow { +public class HbaseAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; @@ -73,13 +73,13 @@ public class HbaseAsyncReqRow extends BaseAsyncReqRow { private transient HBaseClient hBaseClient; - private transient AbstractRowKeyModeDealer rowKeyMode; + private transient AbsRowKeyModeDealer rowKeyMode; private String tableName; private String[] colNames; - public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new HbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -89,7 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List colRefType; - public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java similarity index 87% rename from hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java rename to hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java index 90ee289bd..1506bc440 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase.rowkeydealer; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class AbstractRowKeyModeDealer { +public abstract class AbsRowKeyModeDealer { protected Map colRefType; @@ -60,9 +60,9 @@ public abstract class AbstractRowKeyModeDealer { protected Map sideFieldIndex = Maps.newHashMap(); - public AbstractRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, - boolean openCache, JoinType joinType, List outFieldInfoList, - Map inFieldIndex, Map sideFieldIndex){ + public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, + boolean openCache, JoinType joinType, List outFieldInfoList, + Map inFieldIndex, Map sideFieldIndex){ this.colRefType = colRefType; this.colNames = colNames; this.hBaseClient = hBaseClient; @@ -111,5 +111,5 @@ protected Row fillData(Row input, Object sideInput){ } public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbstractSideCache sideCache); + AbsSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index dcd50131d..a41ffe916 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -54,7 +55,7 @@ * @author xuchao */ -public class PreRowKeyModeDealerDealer extends AbstractRowKeyModeDealer { +public class PreRowKeyModeDealerDealer extends AbsRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(PreRowKeyModeDealerDealer.class); @@ -66,7 +67,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbstractSideCache sideCache) { + AbsSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); prefixScanner.setFilter(scanFilter); @@ -79,7 +80,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF } - private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { @@ -110,7 +111,8 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, for (String key : colNames) { Object val = sideMap.get(key); if (val == null) { - LOG.error("can't get data with column {}", key); + System.out.println("can't get data with column " + key); + LOG.error("can't get data with column " + key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 41208c7f3..b20c316db 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbstractSideCache; +import com.dtstack.flink.sql.side.cache.AbsSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -49,7 +49,7 @@ * @author xuchao */ -public class RowKeyEqualModeDealer extends AbstractRowKeyModeDealer { +public class RowKeyEqualModeDealer extends AbsRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(RowKeyEqualModeDealer.class); @@ -62,7 +62,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbstractSideCache sideCache){ + AbsSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); hBaseClient.get(getRequest).addCallbacks(arg -> { @@ -86,7 +86,8 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF for(String key : colNames){ Object val = sideMap.get(key); if(val == null){ - LOG.error("can't get data with column {}", key); + System.out.println("can't get data with column " + key); + LOG.error("can't get data with column " + key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java index ab826c98f..af5e24dd6 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java @@ -30,7 +30,7 @@ * @author xuchao */ -public class Md5ReplaceOperator extends AbstractReplaceOperator { +public class Md5ReplaceOperator extends ReplaceOperator{ public Md5ReplaceOperator(EReplaceOpType opType) { super(opType); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java similarity index 93% rename from hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java rename to hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java index c75fa43c1..c3c9b5cfe 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java @@ -31,11 +31,11 @@ * @author xuchao */ -public abstract class AbstractReplaceOperator { +public abstract class ReplaceOperator { private EReplaceOpType opType; - public AbstractReplaceOperator(EReplaceOpType opType){ + public ReplaceOperator(EReplaceOpType opType){ this.opType = opType; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java index ac9a3b87c..834edf9ae 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java @@ -28,12 +28,5 @@ */ public enum EReplaceOpType { - /** - * 没有func - */ - NO_FUNC, - /** - * md5 func - */ - MD5_FUNC; + NO_FUNC, MD5_FUNC; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index bb4b1556b..f4b039ef8 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -28,16 +28,7 @@ * @author xuchao */ public enum EReplaceType { - /** - * 参数 - */ PARAM, - /** - * 函数 - */ FUNC, - /** - * 常量 - */ CONSTANT; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index cf44c181f..7b627da4d 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -20,15 +20,15 @@ package com.dtstack.flink.sql.side.hbase.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * hbase field information must include the definition of an alias -> sql which does not allow ":" @@ -37,7 +37,7 @@ * @author xuchao */ -public class HbaseSideParser extends AbstractSideTableParser { +public class HbaseSideParser extends AbsSideTableParser { private final static String FIELD_KEY = "fieldKey"; @@ -58,7 +58,7 @@ public HbaseSideParser() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); @@ -77,7 +77,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class HbaseOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 9308725a2..447b02921 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,6 +30,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -58,7 +59,7 @@ public HbaseSink() { } @Override - public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { HbaseTableInfo hbaseTableInfo = (HbaseTableInfo) targetTableInfo; this.zookeeperQuorum = hbaseTableInfo.getHost(); this.port = hbaseTableInfo.getPort(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 8d8996a23..064b13e69 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -32,14 +32,14 @@ import java.util.List; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Date: 2018/09/14 * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseSinkParser extends AbstractTableParser { +public class HbaseSinkParser extends AbsTableParser { public static final String HBASE_ZOOKEEPER_QUORUM = "zookeeperQuorum"; @@ -60,7 +60,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseTableInfo hbaseTableInfo = new HbaseTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 62f7c9936..610eb5c30 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import java.util.Map; @@ -30,7 +30,7 @@ * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseTableInfo extends AbstractTargetTableInfo { +public class HbaseTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "hbase"; diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 10938308a..9cd8c9194 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -20,10 +20,12 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.JDBCUtils; +import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -34,6 +36,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; +import java.util.Map; /** * side operator with cache for all(period reload) @@ -43,7 +46,7 @@ * @author xiuzhu */ -public class ImpalaAllReqRow extends AbstractRdbAllReqRow { +public class ImpalaAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -53,13 +56,13 @@ public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private ImpalaSideTableInfo impalaSideTableInfo; - public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ImpalaAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.impalaSideTableInfo = (ImpalaSideTableInfo) sideTableInfo; } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Connection connection ; String url = getUrl(); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java index a5e643967..d40c5f48c 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java @@ -20,9 +20,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -31,7 +32,7 @@ public class ImpalaAllSideInfo extends RdbAllSideInfo { - public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -60,7 +61,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); + return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index b5e0ed5fb..725e7c3ee 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import io.vertx.core.Vertx; @@ -50,19 +50,30 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - ImpalaSideTableInfo impalaSideTableInfo = (ImpalaSideTableInfo) sideInfo.getSideTableInfo(); + JsonObject impalaClientConfig = getClientConfig(); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSQLClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + } + public JsonObject getClientConfig() { JsonObject impalaClientConfig = new JsonObject(); impalaClientConfig.put("url", getUrl()) .put("driver_class", IMPALA_DRIVER) - .put("max_pool_size", impalaSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("provider_class", DT_PROVIDER_CLASS) .put("idle_connection_test_period", 300) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) @@ -71,17 +82,9 @@ public void open(Configuration parameters) throws Exception { .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); - System.setProperty("vertx.disableFileCPResolving", "true"); - - VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(impalaSideTableInfo.getAsyncPoolSize()); - vo.setFileResolverCachingEnabled(false); - Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + return impalaClientConfig; } - public String getUrl() { ImpalaSideTableInfo impalaSideTableInfo = (ImpalaSideTableInfo) sideInfo.getSideTableInfo(); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java index 14fbf0ed3..53559b9a7 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -38,7 +38,7 @@ public class ImpalaAsyncSideInfo extends RdbAsyncSideInfo { - public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -69,7 +69,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); + return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index 2f1d73be7..b6a54ca3f 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; @@ -48,7 +48,7 @@ public class ImpalaSideParser extends RdbSideParser { private static final String CURR_TYPE = "impala"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaSideTableInfo impalaSideTableInfo = new ImpalaSideTableInfo(); impalaSideTableInfo.setType(CURR_TYPE); impalaSideTableInfo.setName(tableName); @@ -147,8 +147,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; - default: - break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index cd1e1e945..41fa00926 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,9 +21,9 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +36,7 @@ * @author xiuzhu */ -public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { +public class ImpalaSink extends RdbSink implements IStreamSinkGener { private ImpalaTableInfo impalaTableInfo; @@ -47,7 +47,7 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(getImpalaJdbcUrl()) + .setDBUrl(getImpalaJdbcUrl()) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) @@ -70,8 +70,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbUrl; - StringBuffer urlBuffer = new StringBuffer(dbUrl); + String newUrl = dbURL; + StringBuffer urlBuffer = new StringBuffer(dbURL); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 5b6fdeafe..4921f5e51 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -19,11 +19,12 @@ package com.dtstack.flink.sql.sink.impala.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -41,7 +42,7 @@ public class ImpalaSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "impala"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); @@ -116,8 +117,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; - default: - break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 9958a2544..88c2ca939 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -19,16 +19,16 @@ import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.format.SerializationMetricWrapper; -import com.dtstack.flink.sql.sink.kafka.serialization.AvroCRowSerializationSchema; -import com.dtstack.flink.sql.sink.kafka.serialization.CsvCRowSerializationSchema; -import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.formats.avro.AvroRowSerializationSchema; +import org.apache.flink.formats.csv.CsvRowSerializationSchema; +import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -51,37 +51,42 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); - protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = createSerializationSchema(kafkaSinkTableInfo, typeInformation); - return new SerializationMetricWrapper(serializationSchema); + protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); } - private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = null; + private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = null; if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { + if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonCRowSerializationSchema(typeInformation, kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = new JsonRowSerializationSchema(typeInformation); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } + } else if (FormatType.CSV.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { + if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); - serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); - serSchemaBuilder.setUpdateMode(kafkaSinkTableInfo.getUpdateMode()); + final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(typeInformation); + serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); serializationSchema = serSchemaBuilder.build(); + } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { + if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); + + serializationSchema = new AvroRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java deleted file mode 100644 index 7234216a7..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; -import org.apache.kafka.clients.consumer.ConsumerConfig; - -import java.util.Optional; -import java.util.Properties; -import java.util.stream.IntStream; - -/** - * Date: 2020/4/1 - * Company: www.dtstack.com - * @author maqi - */ -public abstract class AbstractKafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - public static final String SINK_OPERATOR_NAME_TPL = "${topic}_${table}"; - - protected String[] fieldNames; - protected TypeInformation[] fieldTypes; - - protected String[] partitionKeys; - protected String sinkOperatorName; - protected Properties properties; - protected int parallelism; - protected String topic; - protected String tableName; - - protected TableSchema schema; - protected SinkFunction kafkaProducer; - - - protected Optional> partitioner; - - protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { - Properties props = new Properties(); - props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaSinkTableInfo.getBootstrapServers()); - - for (String key : KafkaSinkTableInfo.getKafkaParamKeys()) { - props.setProperty(key, KafkaSinkTableInfo.getKafkaParam(key)); - } - return props; - } - - protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { - Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); - TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) - .toArray(TypeInformation[]::new); - return types; - } - - - protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] fieldTypes) { - Preconditions.checkArgument(fieldNames.length == fieldTypes.length, "fieldNames length must equals fieldTypes length !"); - - TableSchema.Builder builder = TableSchema.builder(); - IntStream.range(0, fieldTypes.length) - .forEach(i -> builder.field(fieldNames[i], fieldTypes[i])); - - return builder.build(); - } - - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream - .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .returns(getRowTypeInfo()) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer).name(sinkOperatorName); - } - - public CRowTypeInfo getRowTypeInfo() { - return new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); - } - - protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index 90dfe996b..e212d1f57 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,13 +11,13 @@ public class CustomerFlinkPartition extends FlinkKafkaPartitioner { public CustomerFlinkPartition() { } - @Override + public void open(int parallelInstanceId, int parallelInstances) { Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); this.parallelInstanceId = parallelInstanceId; } - @Override + public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ @@ -25,11 +25,11 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int } return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } - @Override + public boolean equals(Object o) { return this == o || o instanceof CustomerFlinkPartition; } - @Override + public int hashCode() { return CustomerFlinkPartition.class.hashCode(); } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index cde8d1b1d..498766564 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,20 +2,18 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; -import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; -public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); @@ -32,41 +30,38 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization this.mapper = new ObjectMapper(); } - @Override - public byte[] serializeKey(CRow element) { - if (partitionKeys == null || partitionKeys.length <= 0) { + public byte[] serializeKey(Row element) { + if(partitionKeys == null || partitionKeys.length <=0){ return null; - } - SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); - if (serializationSchema instanceof JsonCRowSerializationSchema) { - return serializeJsonKey((JsonCRowSerializationSchema) serializationSchema, element); + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if(serializationSchema instanceof JsonRowSerializationSchema){ + return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); } return null; } - @Override - public byte[] serializeValue(CRow element) { + public byte[] serializeValue(Row element) { return this.serializationMetricWrapper.serialize(element); } - @Override - public String getTargetTopic(CRow element) { + public String getTargetTopic(Row element) { return null; } - private byte[] serializeJsonKey(JsonCRowSerializationSchema jsonCRowSerializationSchema, CRow element) { + private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { try { - byte[] data = jsonCRowSerializationSchema.serialize(element); + byte[] data = jsonRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); StringBuilder sb = new StringBuilder(); - for (String key : partitionKeys) { - if (objectNode.has(key)) { + for(String key : partitionKeys){ + if(objectNode.has(key)){ sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); - } catch (Exception e) { - if (COUNTER.getAndIncrement() % 1000 == 0) { + } catch (Exception e){ + if(COUNTER.getAndIncrement() % 1000 == 0){ LOG.error("serializeJsonKey error", e); } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java deleted file mode 100644 index 692e208b5..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.serialization; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.util.Utf8; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.TimeZone; -import java.util.stream.Collectors; - -/** - * Serialization schema that serializes CROW into Avro bytes. - * - *

Serializes objects that are represented in (nested) Flink rows. It support types that - * are compatible with Flink's Table & SQL API. - ** - * @author maqi - */ -public class AvroCRowSerializationSchema implements SerializationSchema { - - /** - * Used for time conversions from SQL types. - */ - private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); - - /** - * Avro record class for serialization. Might be null if record class is not available. - */ - private Class recordClazz; - - /** - * Schema string for deserialization. - */ - private String schemaString; - - /** - * Avro serialization schema. - */ - private transient Schema schema; - - /** - * Writer to serialize Avro record into a byte array. - */ - private transient DatumWriter datumWriter; - - /** - * Output stream to serialize records into byte array. - */ - private transient ByteArrayOutputStream arrayOutputStream; - - /** - * Low-level class for serialization of Avro values. - */ - private transient Encoder encoder; - - private String updateMode; - - private String retractKey = "retract"; - - /** - * Creates an Avro serialization schema for the given specific record class. - * - * @param recordClazz Avro record class used to serialize Flink's row to Avro's record - */ - public AvroCRowSerializationSchema(Class recordClazz, String updateMode) { - Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); - this.recordClazz = recordClazz; - this.schema = SpecificData.get().getSchema(recordClazz); - this.schemaString = schema.toString(); - this.datumWriter = new SpecificDatumWriter<>(schema); - this.arrayOutputStream = new ByteArrayOutputStream(); - this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); - this.updateMode = updateMode; - } - - /** - * Creates an Avro serialization schema for the given Avro schema string. - * - * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record - */ - public AvroCRowSerializationSchema(String avroSchemaString,String updateMode) { - Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); - this.recordClazz = null; - this.schemaString = avroSchemaString; - try { - this.schema = new Schema.Parser().parse(avroSchemaString); - } catch (SchemaParseException e) { - throw new IllegalArgumentException("Could not parse Avro schema string.", e); - } - this.datumWriter = new GenericDatumWriter<>(schema); - this.arrayOutputStream = new ByteArrayOutputStream(); - this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); - this.updateMode = updateMode; - } - - @Override - public byte[] serialize(CRow crow) { - try { - Row row = crow.row(); - boolean change = crow.change(); - - // convert to record - final GenericRecord record = convertRowToAvroRecord(schema, row); - - dealRetractField(change, record); - - arrayOutputStream.reset(); - datumWriter.write(record, encoder); - encoder.flush(); - return arrayOutputStream.toByteArray(); - } catch (Exception e) { - throw new RuntimeException("Failed to serialize row.", e); - } - } - - protected void dealRetractField(boolean change, GenericRecord record) { - schema.getFields() - .stream() - .filter(field -> StringUtils.equalsIgnoreCase(field.name(), retractKey)) - .findFirst() - .ifPresent(field -> { - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - record.put(retractKey, convertFlinkType(field.schema(), change)); - } - }); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final AvroCRowSerializationSchema that = (AvroCRowSerializationSchema) o; - return Objects.equals(recordClazz, that.recordClazz) && Objects.equals(schemaString, that.schemaString); - } - - @Override - public int hashCode() { - return Objects.hash(recordClazz, schemaString); - } - - // -------------------------------------------------------------------------------------------- - - private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { - - final List fields = schema.getFields() - .stream() - .filter(field -> !StringUtils.equalsIgnoreCase(field.name(), retractKey)) - .collect(Collectors.toList()); - - final int length = fields.size(); - final GenericRecord record = new GenericData.Record(schema); - for (int i = 0; i < length; i++) { - final Schema.Field field = fields.get(i); - record.put(i, convertFlinkType(field.schema(), row.getField(i))); - } - return record; - } - - private Object convertFlinkType(Schema schema, Object object) { - if (object == null) { - return null; - } - switch (schema.getType()) { - case RECORD: - if (object instanceof Row) { - return convertRowToAvroRecord(schema, (Row) object); - } - throw new IllegalStateException("Row expected but was: " + object.getClass()); - case ENUM: - return new GenericData.EnumSymbol(schema, object.toString()); - case ARRAY: - final Schema elementSchema = schema.getElementType(); - final Object[] array = (Object[]) object; - final GenericData.Array convertedArray = new GenericData.Array<>(array.length, schema); - for (Object element : array) { - convertedArray.add(convertFlinkType(elementSchema, element)); - } - return convertedArray; - case MAP: - final Map map = (Map) object; - final Map convertedMap = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - convertedMap.put( - new Utf8(entry.getKey().toString()), - convertFlinkType(schema.getValueType(), entry.getValue())); - } - return convertedMap; - case UNION: - final List types = schema.getTypes(); - final int size = types.size(); - final Schema actualSchema; - if (size == 2 && types.get(0).getType() == Schema.Type.NULL) { - actualSchema = types.get(1); - } else if (size == 2 && types.get(1).getType() == Schema.Type.NULL) { - actualSchema = types.get(0); - } else if (size == 1) { - actualSchema = types.get(0); - } else { - // generic type - return object; - } - return convertFlinkType(actualSchema, object); - case FIXED: - // check for logical type - if (object instanceof BigDecimal) { - return new GenericData.Fixed( - schema, - convertFromDecimal(schema, (BigDecimal) object)); - } - return new GenericData.Fixed(schema, (byte[]) object); - case STRING: - return new Utf8(object.toString()); - case BYTES: - // check for logical type - if (object instanceof BigDecimal) { - return ByteBuffer.wrap(convertFromDecimal(schema, (BigDecimal) object)); - } - return ByteBuffer.wrap((byte[]) object); - case INT: - // check for logical types - if (object instanceof Date) { - return convertFromDate(schema, (Date) object); - } else if (object instanceof Time) { - return convertFromTime(schema, (Time) object); - } - return object; - case LONG: - // check for logical type - if (object instanceof Timestamp) { - return convertFromTimestamp(schema, (Timestamp) object); - } - return object; - case FLOAT: - case DOUBLE: - case BOOLEAN: - return object; - } - throw new RuntimeException("Unsupported Avro type:" + schema); - } - - private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType instanceof LogicalTypes.Decimal) { - final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; - // rescale to target type - final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); - // byte array must contain the two's-complement representation of the - // unscaled integer value in big-endian byte order - return decimal.unscaledValue().toByteArray(); - } else { - throw new RuntimeException("Unsupported decimal type."); - } - } - - private int convertFromDate(Schema schema, Date date) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType == LogicalTypes.date()) { - // adopted from Apache Calcite - final long time = date.getTime(); - final long converted = time + (long) LOCAL_TZ.getOffset(time); - return (int) (converted / 86400000L); - } else { - throw new RuntimeException("Unsupported date type."); - } - } - - private int convertFromTime(Schema schema, Time date) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType == LogicalTypes.timeMillis()) { - // adopted from Apache Calcite - final long time = date.getTime(); - final long converted = time + (long) LOCAL_TZ.getOffset(time); - return (int) (converted % 86400000L); - } else { - throw new RuntimeException("Unsupported time type."); - } - } - - private long convertFromTimestamp(Schema schema, Timestamp date) { - final LogicalType logicalType = schema.getLogicalType(); - if (logicalType == LogicalTypes.timestampMillis()) { - // adopted from Apache Calcite - final long time = date.getTime(); - return time + (long) LOCAL_TZ.getOffset(time); - } else { - throw new RuntimeException("Unsupported timestamp type."); - } - } - - private void writeObject(ObjectOutputStream outputStream) throws IOException { - outputStream.writeObject(recordClazz); - outputStream.writeObject(schemaString); // support for null - outputStream.writeObject(retractKey); - outputStream.writeObject(updateMode); - } - - @SuppressWarnings("unchecked") - private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { - recordClazz = (Class) inputStream.readObject(); - schemaString = (String) inputStream.readObject(); - if (recordClazz != null) { - schema = SpecificData.get().getSchema(recordClazz); - } else { - schema = new Schema.Parser().parse(schemaString); - } - retractKey = (String) inputStream.readObject(); - updateMode = (String) inputStream.readObject(); - - datumWriter = new SpecificDatumWriter<>(schema); - arrayOutputStream = new ByteArrayOutputStream(); - encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); - } -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java deleted file mode 100644 index 4e57b6f2a..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.serialization; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.formats.csv.CsvRowDeserializationSchema; -import org.apache.flink.formats.csv.CsvRowSchemaConverter; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.io.Serializable; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Objects; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -/** - * Serialization schema that serializes an object of Flink types into a CSV bytes. - * - *

Serializes the input row into a {@link ObjectNode} and - * converts it into byte[]. - * - *

Result byte[] messages can be deserialized using {@link CsvRowDeserializationSchema}. - */ -@PublicEvolving -public final class CsvCRowSerializationSchema implements SerializationSchema { - - private static final long serialVersionUID = 2098447220136965L; - - /** Type information describing the input CSV data. */ - private final RowTypeInfo typeInfo; - - /** Runtime instance that performs the actual work. */ - private final RuntimeConverter runtimeConverter; - - /** CsvMapper used to write {@link JsonNode} into bytes. */ - private final CsvMapper csvMapper; - - /** Schema describing the input CSV data. */ - private CsvSchema csvSchema; - - /** Object writer used to write rows. It is configured by {@link CsvSchema}. */ - private ObjectWriter objectWriter; - - /** Reusable object node. */ - private transient ObjectNode root; - - private String updateMode; - - private String retractKey = "retract"; - - private CsvCRowSerializationSchema( - RowTypeInfo typeInfo, - CsvSchema csvSchema, - String updateMode) { - this.typeInfo = typeInfo; - this.runtimeConverter = createRowRuntimeConverter(typeInfo, true); - this.csvMapper = new CsvMapper(); - this.csvSchema = csvSchema; - this.updateMode = updateMode; - this.objectWriter = csvMapper.writer(csvSchema); - } - - /** - * A builder for creating a {@link CsvRowSerializationSchema}. - */ - @PublicEvolving - public static class Builder { - - private final RowTypeInfo typeInfo; - private CsvSchema csvSchema; - private String updateMode; - - /** - * Creates a {@link CsvRowSerializationSchema} expecting the given {@link TypeInformation}. - * - * @param typeInfo type information used to create schema. - */ - public Builder(TypeInformation typeInfo) { - Preconditions.checkNotNull(typeInfo, "Type information must not be null."); - - if (!(typeInfo instanceof CRowTypeInfo)) { - throw new IllegalArgumentException("Row type information expected."); - } - RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); - this.typeInfo = rowTypeInfo; - this.csvSchema = CsvRowSchemaConverter.convert(rowTypeInfo); - } - - public Builder setFieldDelimiter(char c) { - this.csvSchema = this.csvSchema.rebuild().setColumnSeparator(c).build(); - return this; - } - - public Builder setLineDelimiter(String delimiter) { - Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { - throw new IllegalArgumentException( - "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); - } - this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); - return this; - } - - public Builder setArrayElementDelimiter(String delimiter) { - Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - this.csvSchema = this.csvSchema.rebuild().setArrayElementSeparator(delimiter).build(); - return this; - } - - public Builder setQuoteCharacter(char c) { - this.csvSchema = this.csvSchema.rebuild().setQuoteChar(c).build(); - return this; - } - - public Builder setEscapeCharacter(char c) { - this.csvSchema = this.csvSchema.rebuild().setEscapeChar(c).build(); - return this; - } - - public Builder setNullLiteral(String s) { - this.csvSchema = this.csvSchema.rebuild().setNullValue(s).build(); - return this; - } - - public Builder setUpdateMode(String updateMode) { - this.updateMode = updateMode; - return this; - } - - public CsvCRowSerializationSchema build() { - return new CsvCRowSerializationSchema( - typeInfo, - csvSchema, - updateMode); - } - } - - @Override - public byte[] serialize(CRow crow) { - Row row = crow.row(); - boolean change = crow.change(); - if (root == null) { - root = csvMapper.createObjectNode(); - } - try { - runtimeConverter.convert(csvMapper, root, row); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - fillRetractField(row, change); - } - - return objectWriter.writeValueAsBytes(root); - } catch (Throwable t) { - throw new RuntimeException("Could not serialize row '" + row + "'.", t); - } - } - - protected void fillRetractField(Row row, boolean change) { - root.put(retractKey, change); - CsvSchema.Builder newBuilder = new CsvSchema.Builder(csvSchema); - - CsvSchema.Column retractColumn = new CsvSchema.Column(row.getArity(), retractKey, CsvSchema.ColumnType.BOOLEAN); - newBuilder.addColumn(retractColumn); - csvSchema = newBuilder.build(); - - this.objectWriter = csvMapper.writer(csvSchema); - - } - - @Override - public boolean equals(Object o) { - if (o == null || o.getClass() != this.getClass()) { - return false; - } - if (this == o) { - return true; - } - final CsvCRowSerializationSchema that = (CsvCRowSerializationSchema) o; - final CsvSchema otherSchema = that.csvSchema; - - return typeInfo.equals(that.typeInfo) && - csvSchema.getColumnSeparator() == otherSchema.getColumnSeparator() && - Arrays.equals(csvSchema.getLineSeparator(), otherSchema.getLineSeparator()) && - csvSchema.getArrayElementSeparator().equals(otherSchema.getArrayElementSeparator()) && - csvSchema.getQuoteChar() == otherSchema.getQuoteChar() && - csvSchema.getEscapeChar() == otherSchema.getEscapeChar() && - Arrays.equals(csvSchema.getNullValue(), otherSchema.getNullValue()); - } - - @Override - public int hashCode() { - return Objects.hash( - typeInfo, - csvSchema.getColumnSeparator(), - csvSchema.getLineSeparator(), - csvSchema.getArrayElementSeparator(), - csvSchema.getQuoteChar(), - csvSchema.getEscapeChar(), - csvSchema.getNullValue()); - } - - // -------------------------------------------------------------------------------------------- - - private interface RuntimeConverter extends Serializable { - JsonNode convert(CsvMapper csvMapper, ContainerNode container, Object obj); - } - - private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) { - final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes(); - final String[] fieldNames = rowTypeInfo.getFieldNames(); - - final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes); - - return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters); - } - - private static RuntimeConverter[] createFieldRuntimeConverters(TypeInformation[] fieldTypes) { - final RuntimeConverter[] fieldConverters = new RuntimeConverter[fieldTypes.length]; - for (int i = 0; i < fieldTypes.length; i++) { - fieldConverters[i] = createNullableRuntimeConverter(fieldTypes[i]); - } - return fieldConverters; - } - - private static RuntimeConverter assembleRowRuntimeConverter( - boolean isTopLevel, - String[] fieldNames, - RuntimeConverter[] fieldConverters) { - final int rowArity = fieldNames.length; - // top level reuses the object node container - if (isTopLevel) { - return (csvMapper, container, obj) -> { - final Row row = (Row) obj; - - validateArity(rowArity, row.getArity()); - - final ObjectNode objectNode = (ObjectNode) container; - for (int i = 0; i < rowArity; i++) { - objectNode.set( - fieldNames[i], - fieldConverters[i].convert(csvMapper, container, row.getField(i))); - } - return objectNode; - }; - } else { - return (csvMapper, container, obj) -> { - final Row row = (Row) obj; - - validateArity(rowArity, row.getArity()); - - final ArrayNode arrayNode = csvMapper.createArrayNode(); - for (int i = 0; i < rowArity; i++) { - arrayNode.add(fieldConverters[i].convert(csvMapper, arrayNode, row.getField(i))); - } - return arrayNode; - }; - } - } - - private static RuntimeConverter createNullableRuntimeConverter(TypeInformation info) { - final RuntimeConverter valueConverter = createRuntimeConverter(info); - return (csvMapper, container, obj) -> { - if (obj == null) { - return container.nullNode(); - } - return valueConverter.convert(csvMapper, container, obj); - }; - } - - private static RuntimeConverter createRuntimeConverter(TypeInformation info) { - if (info.equals(Types.VOID)) { - return (csvMapper, container, obj) -> container.nullNode(); - } else if (info.equals(Types.STRING)) { - return (csvMapper, container, obj) -> container.textNode((String) obj); - } else if (info.equals(Types.BOOLEAN)) { - return (csvMapper, container, obj) -> container.booleanNode((Boolean) obj); - } else if (info.equals(Types.BYTE)) { - return (csvMapper, container, obj) -> container.numberNode((Byte) obj); - } else if (info.equals(Types.SHORT)) { - return (csvMapper, container, obj) -> container.numberNode((Short) obj); - } else if (info.equals(Types.INT)) { - return (csvMapper, container, obj) -> container.numberNode((Integer) obj); - } else if (info.equals(Types.LONG)) { - return (csvMapper, container, obj) -> container.numberNode((Long) obj); - } else if (info.equals(Types.FLOAT)) { - return (csvMapper, container, obj) -> container.numberNode((Float) obj); - } else if (info.equals(Types.DOUBLE)) { - return (csvMapper, container, obj) -> container.numberNode((Double) obj); - } else if (info.equals(Types.BIG_DEC)) { - return (csvMapper, container, obj) -> container.numberNode((BigDecimal) obj); - } else if (info.equals(Types.BIG_INT)) { - return (csvMapper, container, obj) -> container.numberNode((BigInteger) obj); - } else if (info.equals(Types.SQL_DATE)) { - return (csvMapper, container, obj) -> container.textNode(obj.toString()); - } else if (info.equals(Types.SQL_TIME)) { - return (csvMapper, container, obj) -> container.textNode(obj.toString()); - } else if (info.equals(Types.SQL_TIMESTAMP)) { - return (csvMapper, container, obj) -> container.textNode(obj.toString()); - } else if (info instanceof RowTypeInfo){ - return createRowRuntimeConverter((RowTypeInfo) info, false); - } else if (info instanceof BasicArrayTypeInfo) { - return createObjectArrayRuntimeConverter(((BasicArrayTypeInfo) info).getComponentInfo()); - } else if (info instanceof ObjectArrayTypeInfo) { - return createObjectArrayRuntimeConverter(((ObjectArrayTypeInfo) info).getComponentInfo()); - } else if (info instanceof PrimitiveArrayTypeInfo && - ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { - return createByteArrayRuntimeConverter(); - } - else { - throw new RuntimeException("Unsupported type information '" + info + "'."); - } - } - - private static RuntimeConverter createObjectArrayRuntimeConverter(TypeInformation elementType) { - final RuntimeConverter elementConverter = createNullableRuntimeConverter(elementType); - return (csvMapper, container, obj) -> { - final Object[] array = (Object[]) obj; - final ArrayNode arrayNode = csvMapper.createArrayNode(); - for (Object element : array) { - arrayNode.add(elementConverter.convert(csvMapper, arrayNode, element)); - } - return arrayNode; - }; - } - - private static RuntimeConverter createByteArrayRuntimeConverter() { - return (csvMapper, container, obj) -> container.binaryNode((byte[]) obj); - } - - private static void validateArity(int expected, int actual) { - if (expected != actual) { - throw new RuntimeException("Row length mismatch. " + expected + - " fields expected but was " + actual + "."); - } - } -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java deleted file mode 100644 index bfe801d52..000000000 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.serialization; - -import com.dtstack.flink.sql.enums.EUpdateMode; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.formats.json.JsonRowSchemaConverter; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.runtime.types.CRowTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Time; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.util.Objects; - -/** - * - * Serialization schema that serializes an object of Flink types into a JSON bytes. - * - *

Serializes the input Flink object into a JSON string and - * converts it into byte[]. - * - */ -public class JsonCRowSerializationSchema implements SerializationSchema { - - private static final long serialVersionUID = -2885556750743978636L; - - /** Type information describing the input type. */ - private final TypeInformation typeInfo; - - /** Object mapper that is used to create output JSON objects. */ - private final ObjectMapper mapper = new ObjectMapper(); - - /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ - private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); - - /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ - private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); - - /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ - private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - - /** Reusable object node. */ - private transient ObjectNode node; - - private String updateMode; - - private final String retractKey = "retract"; - - public JsonCRowSerializationSchema(String jsonSchema, String updateMode) { - this(JsonRowSchemaConverter.convert(jsonSchema), updateMode); - } - - /** - * Creates a JSON serialization schema for the given type information. - * - * @param typeInfo The field names of {@link Row} are used to map to JSON properties. - */ - public JsonCRowSerializationSchema(TypeInformation typeInfo, String updateMode) { - Preconditions.checkNotNull(typeInfo, "Type information"); - this.typeInfo = typeInfo; - this.updateMode = updateMode; - } - - - @Override - public byte[] serialize(CRow crow) { - Row row = crow.row(); - boolean change = crow.change(); - if (node == null) { - node = mapper.createObjectNode(); - } - - RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); - try { - convertRow(node, rowTypeInfo, row); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - node.put(retractKey, change); - } - return mapper.writeValueAsBytes(node); - } catch (Throwable t) { - throw new RuntimeException("Could not serialize row '" + row + "'. " + - "Make sure that the schema matches the input.", t); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final JsonCRowSerializationSchema that = (JsonCRowSerializationSchema) o; - return Objects.equals(typeInfo, that.typeInfo); - } - - @Override - public int hashCode() { - return Objects.hash(typeInfo); - } - - // -------------------------------------------------------------------------------------------- - - private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { - if (reuse == null) { - reuse = mapper.createObjectNode(); - } - final String[] fieldNames = info.getFieldNames(); - - final TypeInformation[] fieldTypes = info.getFieldTypes(); - - // validate the row - if (row.getArity() != fieldNames.length) { - throw new IllegalStateException(String.format( - "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); - } - - for (int i = 0; i < fieldNames.length; i++) { - final String name = fieldNames[i]; - - final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); - reuse.set(name, fieldConverted); - } - - return reuse; - } - - private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { - if (info == Types.VOID || object == null) { - return container.nullNode(); - } else if (info == Types.BOOLEAN) { - return container.booleanNode((Boolean) object); - } else if (info == Types.STRING) { - return container.textNode((String) object); - } else if (info == Types.BIG_DEC) { - // convert decimal if necessary - if (object instanceof BigDecimal) { - return container.numberNode((BigDecimal) object); - } - return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); - } else if (info == Types.BIG_INT) { - // convert integer if necessary - if (object instanceof BigInteger) { - return container.numberNode((BigInteger) object); - } - return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); - } else if (info == Types.SQL_DATE) { - return container.textNode(object.toString()); - } else if (info == Types.SQL_TIME) { - final Time time = (Time) object; - // strip milliseconds if possible - if (time.getTime() % 1000 > 0) { - return container.textNode(timeFormatWithMillis.format(time)); - } - return container.textNode(timeFormat.format(time)); - } else if (info == Types.SQL_TIMESTAMP) { - return container.textNode(timestampFormat.format((Timestamp) object)); - } else if (info instanceof RowTypeInfo) { - if (reuse != null && reuse instanceof ObjectNode) { - return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); - } else { - return convertRow(null, (RowTypeInfo) info, (Row) object); - } - } else if (info instanceof ObjectArrayTypeInfo) { - if (reuse != null && reuse instanceof ArrayNode) { - return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } else { - return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } - } else if (info instanceof BasicArrayTypeInfo) { - if (reuse != null && reuse instanceof ArrayNode) { - return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } else { - return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } - } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { - return container.binaryNode((byte[]) object); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return mapper.valueToTree(object); - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); - } - } - } - - private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { - if (reuse == null) { - reuse = mapper.createArrayNode(); - } else { - reuse.removeAll(); - } - - for (Object object : array) { - reuse.add(convert(reuse, null, info, object)); - } - return reuse; - } -} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4ad8947a8..8ba21b953 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -18,10 +18,9 @@ package com.dtstack.flink.sql.sink.kafka.table; -import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,13 +32,12 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkParser extends AbstractTableParser { +public class KafkaSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); kafkaSinkTableInfo.setName(tableName); kafkaSinkTableInfo.setType(MathUtil.getString(props.get(KafkaSinkTableInfo.TYPE_KEY.toLowerCase()))); - parseFieldsInfo(fieldsInfo, kafkaSinkTableInfo); if (props.get(KafkaSinkTableInfo.SINK_DATA_TYPE) != null) { @@ -48,14 +46,11 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -70,8 +59,6 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String partitionKeys; - private String updateMode; - public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -84,6 +71,7 @@ public Set getKafkaParamKeys() { return kafkaParam.keySet(); } + public String getBootstrapServers() { return bootstrapServers; } @@ -116,40 +104,14 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } - public String getUpdateMode() { - return updateMode; - } - - public void setUpdateMode(String updateMode) { - this.updateMode = updateMode; - } - @Override public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); - - if (StringUtils.equalsIgnoreCase(getSinkDataType(), FormatType.AVRO.name())) { - avroParamCheck(); - } - return false; } - public void avroParamCheck() { - Preconditions.checkNotNull(schemaString, "avro type schemaInfo is required"); - if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - Schema schema = new Schema.Parser().parse(schemaString); - schema.getFields() - .stream() - .filter(field -> StringUtils.equalsIgnoreCase(field.name(), RETRACT_FIELD_KEY)) - .findFirst() - .orElseThrow(() -> - new NullPointerException(String.valueOf("arvo upsert mode the retract attribute must be contained in schemaInfo field "))); - } - } - public String getEnableKeyPartition() { return enableKeyPartition; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java deleted file mode 100644 index 852a381e2..000000000 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka; - -import com.dtstack.flink.sql.source.IStreamSourceGener; -import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; -import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; -import org.apache.flink.table.api.Table; -import org.apache.flink.types.Row; -import org.apache.kafka.clients.consumer.ConsumerConfig; - -import java.util.Map; -import java.util.Properties; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -/** - * Date: 2020/3/20 - * Company: www.dtstack.com - * @author maqi - */ -public abstract class AbstractKafkaSource implements IStreamSourceGener

{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - protected Properties getKafkaProperties(KafkaSourceTableInfo kafkaSourceTableInfo) { - Properties props = new Properties(); - props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSourceTableInfo.getBootstrapServers()); - - if (DtStringUtil.isJson(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, EKafkaOffset.NONE.name().toLowerCase()); - } else { - props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceTableInfo.getOffsetReset()); - } - - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceTableInfo.getGroupId()); - } - - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - return props; - } - - protected String generateOperatorName(String tabName, String topicName) { - return SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", tabName); - } - - protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { - Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); - TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) - .toArray(TypeInformation[]::new); - - return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - } - - protected void setStartPosition(String offset, String topicName, FlinkKafkaConsumerBase kafkaSrc) { - if (StringUtils.equalsIgnoreCase(offset, EKafkaOffset.EARLIEST.name())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJson(offset)) { - Map specificStartupOffsets = buildOffsetMap(offset, topicName); - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } else { - kafkaSrc.setStartFromLatest(); - } - } - - /** - * kafka offset,eg.. {"0":12312,"1":12321,"2":12312} - * @param offsetJson - * @param topicName - * @return - */ - protected Map buildOffsetMap(String offsetJson, String topicName) { - try { - Properties properties = PluginUtil.jsonStrToObject(offsetJson, Properties.class); - Map offsetMap = PluginUtil.objectToMap(properties); - Map specificStartupOffsets = offsetMap - .entrySet() - .stream() - .collect(Collectors.toMap( - (Map.Entry entry) -> new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), - (Map.Entry entry) -> Long.valueOf(entry.getValue().toString())) - ); - - return specificStartupOffsets; - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + offsetJson); - } - } - -} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java index f08287cec..afa950c5b 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaDeserializationMetricWrapper.java @@ -77,7 +77,7 @@ protected void beforeDeserialize() throws IOException { protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - Field consumerThreadField = getConsumerThreadField(fetcher); + Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); consumerThreadField.setAccessible(true); KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); @@ -115,14 +115,6 @@ public Long getValue() { } } - private Field getConsumerThreadField(AbstractFetcher fetcher) throws NoSuchFieldException { - try { - return fetcher.getClass().getDeclaredField("consumerThread"); - } catch (Exception e) { - return fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); - } - } - public void setFetcher(AbstractFetcher fetcher) { this.fetcher = fetcher; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index bef86f10d..005bfb4c2 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,14 +19,11 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; -import com.dtstack.flink.sql.table.AbstractSourceParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import java.util.stream.Collectors; /** * Reason: @@ -35,38 +32,33 @@ * * @author sishu.yss */ -public class KafkaSourceParser extends AbstractSourceParser { +public class KafkaSourceParser extends AbsSourceParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); - kafkaSourceTableInfo.setName(tableName); kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); + parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); + String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); + if (bootstrapServer == null || bootstrapServer.trim().equals("")) { + throw new Exception("BootstrapServers can not be empty!"); + } else { + kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); + } kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()), false)); - kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase(), EKafkaOffset.LATEST.name().toLowerCase()))); - kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - - kafkaSourceTableInfo.setSchemaString(MathUtil.getString(props.get(KafkaSourceTableInfo.SCHEMA_STRING_KEY.toLowerCase()))); - kafkaSourceTableInfo.setFieldDelimiter(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.CSV_FIELD_DELIMITER_KEY.toLowerCase(), "|"))); - kafkaSourceTableInfo.setSourceDataType(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.SOURCE_DATA_TYPE_KEY.toLowerCase(), FormatType.DT_NEST.name()))); - - Map kafkaParams = props.keySet().stream() - .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) - .collect(Collectors.toMap( - key -> key.substring(6), key -> props.get(key).toString()) - ); - - kafkaSourceTableInfo.addKafkaParam(kafkaParams); + for (String key : props.keySet()) { + if (!key.isEmpty() && key.startsWith("kafka.")) { + kafkaSourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); + } + } kafkaSourceTableInfo.check(); - return kafkaSourceTableInfo; } } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index c27eee376..4f4851335 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -17,10 +17,11 @@ */ + package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -34,126 +35,136 @@ * @author sishu.yss */ -public class KafkaSourceTableInfo extends AbstractSourceTableInfo { - - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; +public class KafkaSourceTableInfo extends SourceTableInfo { - public static final String TOPIC_KEY = "topic"; + public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - public static final String TYPE_KEY = "type"; + public static final String TOPIC_KEY = "topic"; - public static final String GROUPID_KEY = "groupId"; + public static final String TYPE_KEY = "type"; - public static final String OFFSETRESET_KEY = "offsetReset"; + public static final String GROUPID_KEY = "groupId"; - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String OFFSETRESET_KEY = "offsetReset"; - public static final String SCHEMA_STRING_KEY = "schemaInfo"; + public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - public static final String CSV_FIELD_DELIMITER_KEY = "fieldDelimiter"; + private String bootstrapServers; - public static final String SOURCE_DATA_TYPE_KEY = "sourceDataType"; + private String topic; - private String bootstrapServers; + private String groupId; - private String topic; + //latest, earliest + private String offsetReset = "latest"; - private String groupId; + private String offset; - private String offsetReset; + private Boolean topicIsPattern = false; - private Boolean topicIsPattern = false; + private String sourceDataType = FormatType.DT_NEST.name(); - private String sourceDataType; + private String schemaString; - private String schemaString; + private String fieldDelimiter; - private String fieldDelimiter; + public String getBootstrapServers() { + return bootstrapServers; + } - public Map kafkaParam = new HashMap<>(); + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } + public String getTopic() { + return topic; + } - public String getBootstrapServers() { - return bootstrapServers; - } + public void setTopic(String topic) { + this.topic = topic; + } - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } + public String getGroupId() { + return groupId; + } - public String getTopic() { - return topic; - } + public void setGroupId(String groupId) { + this.groupId = groupId; + } - public void setTopic(String topic) { - this.topic = topic; - } + public String getOffsetReset() { + return offsetReset; + } - public String getGroupId() { - return groupId; - } + public void setOffsetReset(String offsetReset) { + if(offsetReset == null){ + return; + } + this.offsetReset = offsetReset; + } - public void setGroupId(String groupId) { - this.groupId = groupId; - } + public String getOffset() { + return offset; + } - public String getOffsetReset() { - return offsetReset; - } + public void setOffset(String offset) { + if (offsetReset == null) { + return; + } + this.offset = offset; + } - public void setOffsetReset(String offsetReset) { - this.offsetReset = offsetReset; - } + public Boolean getTopicIsPattern() { + return topicIsPattern; + } - public Boolean getTopicIsPattern() { - return topicIsPattern; - } + public void setTopicIsPattern(Boolean topicIsPattern) { + this.topicIsPattern = topicIsPattern; + } - public void setTopicIsPattern(Boolean topicIsPattern) { - this.topicIsPattern = topicIsPattern; - } + public Map kafkaParam = new HashMap<>(); - public void addKafkaParam(Map kafkaParam) { - kafkaParam.putAll(kafkaParam); - } + public void addKafkaParam(String key, String value) { + kafkaParam.put(key, value); + } - public String getKafkaParam(String key) { - return kafkaParam.get(key); - } + public String getKafkaParam(String key) { + return kafkaParam.get(key); + } - public Set getKafkaParamKeys() { - return kafkaParam.keySet(); - } + public Set getKafkaParamKeys() { + return kafkaParam.keySet(); + } - public String getSourceDataType() { - return sourceDataType; - } + public String getSourceDataType() { + return sourceDataType; + } - public void setSourceDataType(String sourceDataType) { - this.sourceDataType = sourceDataType; - } + public void setSourceDataType(String sourceDataType) { + this.sourceDataType = sourceDataType; + } - public String getSchemaString() { - return schemaString; - } + public String getSchemaString() { + return schemaString; + } - public void setSchemaString(String schemaString) { - this.schemaString = schemaString; - } + public void setSchemaString(String schemaString) { + this.schemaString = schemaString; + } - public String getFieldDelimiter() { - return fieldDelimiter; - } + public String getFieldDelimiter() { + return fieldDelimiter; + } - public void setFieldDelimiter(String fieldDelimiter) { - this.fieldDelimiter = fieldDelimiter; - } + public void setFieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 1cbbeafd9..71a9cc386 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer extends FlinkKafkaProducer { +public class KafkaProducer extends FlinkKafkaProducer { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); @@ -45,7 +45,7 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index 6bf9014df..f3a2f40f5 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 632bb720e..d0d477f4e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,23 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -29,23 +44,104 @@ * @create: 2019-11-05 11:45 * @description: **/ -public class KafkaSink extends AbstractKafkaSink { +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected int parallelism; + + protected Properties properties; + + protected FlinkKafkaProducer flinkKafkaProducer; + + /** The schema of the table. */ + private TableSchema schema; + + /** Partitioner to select Kafka partition for each item. */ + protected Optional> partitioner; + + private String[] partitionKeys; + @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - - Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); - this.tableName = kafkaSinkTableInfo.getName(); this.topic = kafkaSinkTableInfo.getTopic(); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); + + for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); + } this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafkaSinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafkaSinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; + for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } } diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 394ea86ee..a445a6529 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -18,17 +18,24 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -36,24 +43,75 @@ * @create: 2019-11-05 10:55 * @description: **/ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } + + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + + FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + + + //earliest,latest + if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for (Map.Entry entry : offsetMap.entrySet()) { + specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + } else { + kafkaSrc.setStartFromLatest(); + } - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } return tableEnv.fromDataStream(kafkaSource, fields); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index bee1865dd..c815e134a 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -23,7 +23,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer09 extends FlinkKafkaProducer09 { +public class KafkaProducer09 extends FlinkKafkaProducer09 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); @@ -46,7 +45,7 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index ee3423b07..7fb3909ee 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -37,7 +36,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d22be3d59..52caa7bd2 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,23 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -31,22 +46,109 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink extends AbstractKafkaSink{ +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected Properties properties; + + protected FlinkKafkaProducer09 kafkaProducer09; + + /** The schema of the table. */ + private TableSchema schema; + + /** Partitioner to select Kafka partition for each item. */ + protected Optional> partitioner; + + private String[] partitionKeys; + + protected int parallelism; + + + @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - - Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); - this.tableName = kafka09SinkTableInfo.getName(); this.topic = kafka09SinkTableInfo.getTopic(); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); + for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); + } + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafka09SinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafka09SinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducer09Factory().createKafkaProducer(kafka09SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; + for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafka09SinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i) new KafkaProducer09Factory() + .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer09) + .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 9f8917761..21ed7c3e3 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -17,13 +17,17 @@ */ + package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; @@ -32,6 +36,7 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -42,23 +47,74 @@ * @author xuchao */ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } + // only required for Kafka 0.8 + //TODO props.setProperty("zookeeper.connect", kafkaSourceTableInfo.) - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for(int i = 0; i< kafkaSourceTableInfo.getFieldClasses().length; i++){ + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } + + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + + //earliest,latest + if("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())){ + kafkaSrc.setStartFromEarliest(); + }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for(Map.Entry entry:offsetMap.entrySet()){ + specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + }else { + kafkaSrc.setStartFromLatest(); + } - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } return tableEnv.fromDataStream(kafkaSource, fields); } } \ No newline at end of file diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3936575ef..3cdc45dec 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer010 extends FlinkKafkaProducer010 { +public class KafkaProducer010 extends FlinkKafkaProducer010 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); @@ -45,7 +45,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index e0e023586..c44a9fe86 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -37,7 +36,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index eea78e121..42f1045bf 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,21 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -33,22 +46,105 @@ * @modifyer maqi * */ -public class KafkaSink extends AbstractKafkaSink { - @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { - KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected Properties properties; - Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); - this.tableName = kafka10SinkTableInfo.getName(); + protected int parallelism; + + protected KafkaSinkTableInfo kafka10SinkTableInfo; + + /** The schema of the table. */ + private TableSchema schema; + + private String[] partitionKeys; + + @Override + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka10SinkTableInfo.getTopic(); - this.partitioner = Optional.of(new CustomerFlinkPartition<>()); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); + + for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); + } + this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafka10SinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafka10SinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; + for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafka10SinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + + RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, + Optional.of(new CustomerFlinkPartition<>()), partitionKeys); + + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 23989ab7e..55cffbf1e 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,17 +19,24 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -40,24 +47,74 @@ * @author sishu.yss */ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") + @Override + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } - return tableEnv.fromDataStream(kafkaSource, fields); - } + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } + + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + + FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + + //earliest,latest + if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for (Map.Entry entry : offsetMap.entrySet()) { + specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + } else { + kafkaSrc.setStartFromLatest(); + } + + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } + return tableEnv.fromDataStream(kafkaSource, fields); + } } \ No newline at end of file diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 429d21a79..7880dd419 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer011 extends FlinkKafkaProducer011 { +public class KafkaProducer011 extends FlinkKafkaProducer011 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); @@ -46,7 +46,7 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index 0cb11da82..e2272b16e 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -37,7 +36,7 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ea45280c7..c5da782d0 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,8 +18,23 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.types.Row; import java.util.Optional; import java.util.Properties; @@ -34,22 +49,106 @@ * @modifyer maqi * */ -public class KafkaSink extends AbstractKafkaSink { +public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + protected String[] fieldNames; + + protected TypeInformation[] fieldTypes; + + protected String topic; + + protected int parallelism; + + protected Properties properties; + + protected FlinkKafkaProducer011 kafkaProducer011; + + /** The schema of the table. */ + private TableSchema schema; + + /** Partitioner to select Kafka partition for each item. */ + protected Optional> partitioner; + private String[] partitionKeys; + + @Override - public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - - Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); - this.tableName = kafka11SinkTableInfo.getName(); this.topic = kafka11SinkTableInfo.getTopic(); + + properties = new Properties(); + properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); + + for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { + properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); + } this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); - this.fieldTypes = getTypeInformations(kafka11SinkTableInfo); - this.schema = buildTableSchema(fieldNames, fieldTypes); - this.parallelism = kafka11SinkTableInfo.getParallelism(); - this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); - this.kafkaProducer = new KafkaProducer011Factory().createKafkaProducer(kafka11SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); + TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; + for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafka11SinkTableInfo.getFieldClasses()[i]); + } + this.fieldTypes = types; + + TableSchema.Builder schemaBuilder = TableSchema.builder(); + for (int i=0;i) new KafkaProducer011Factory() + .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); return this; } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) + .map((Tuple2 record) -> record.f1) + .returns(getOutputType().getTypeAt(1)) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ + if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java index c20f0678b..3f804fc6c 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java @@ -31,8 +31,8 @@ /** * company: www.dtstack.com + * author: toutian * create: 2019/12/24 - * @author: toutian */ public class KafkaConsumer011Factory extends AbstractKafkaConsumerFactory { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index f58d59d05..1a4eed08a 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,16 +19,24 @@ package com.dtstack.flink.sql.source.kafka; +import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; + +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -39,24 +47,76 @@ * @author sishu.yss */ -public class KafkaSource extends AbstractKafkaSource { +public class KafkaSource implements IStreamSourceGener
{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + /** + * Get kafka data source, you need to provide the data field names, data types + * If you do not specify auto.offset.reset, the default use groupoffset + * + * @param sourceTableInfo + * @return + */ + @SuppressWarnings("rawtypes") + @Override + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); + + Properties props = new Properties(); + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); + if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ + props.setProperty("auto.offset.reset", "none"); + } else { + props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); + } + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ + props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); + } + // only required for Kafka 0.8 + //TODO props.setProperty("zookeeper.connect", kafka09SourceTableInfo.) + + TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; + for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { + types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); + } - @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); - TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); - FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); + FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + //earliest,latest + if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} + try { + Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); + Map offsetMap = PluginUtil.ObjectToMap(properties); + Map specificStartupOffsets = new HashMap<>(); + for (Map.Entry entry : offsetMap.entrySet()) { + specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); + } + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); + } + } else { + kafkaSrc.setStartFromLatest(); + } - setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); + String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - return tableEnv.fromDataStream(kafkaSource, fields); - } + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + Integer parallelism = kafkaSourceTableInfo.getParallelism(); + if (parallelism != null) { + kafkaSource.setParallelism(parallelism); + } + return tableEnv.fromDataStream(kafkaSource, fields); + } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 31f9aa13d..63ff31c50 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,23 +1,24 @@ package com.dtstack.flink.sql.side.kudu; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; +import org.apache.flink.util.Preconditions; + +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -40,7 +41,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -public class KuduAllReqRow extends BaseAllReqRow { +public class KuduAllReqRow extends AllReqRow { private static final long serialVersionUID = 6051774809356082219L; @@ -61,7 +62,7 @@ public class KuduAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new KuduAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -228,6 +229,9 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { if (null != workerCount) { kuduClientBuilder.workerCount(workerCount); } + if (null != defaultSocketReadTimeoutMs) { + kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); + } if (null != defaultOperationTimeoutMs) { kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java index 1241a6f37..ddfa73201 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -12,16 +12,16 @@ import java.util.List; -public class KuduAllSideInfo extends BaseSideInfo { +public class KuduAllSideInfo extends SideInfo { private static final long serialVersionUID = 9005389633060174746L; - public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { // no use } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e534a85f9..e1705d796 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,12 +1,20 @@ package com.dtstack.flink.sql.side.kudu; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -16,13 +24,6 @@ import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -41,7 +42,7 @@ import java.util.List; import java.util.Map; -public class KuduAsyncReqRow extends BaseAsyncReqRow { +public class KuduAsyncReqRow extends AsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); /** @@ -64,7 +65,7 @@ public class KuduAsyncReqRow extends BaseAsyncReqRow { private AsyncKuduScanner.AsyncKuduScannerBuilder scannerBuilder; - public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new KuduAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -94,6 +95,9 @@ private void connKuDu() throws KuduException { if (null != workerCount) { asyncKuduClientBuilder.workerCount(workerCount); } + if (null != defaultSocketReadTimeoutMs) { + asyncKuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); + } if (null != defaultOperationTimeoutMs) { asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index 5ace515f7..de75f23fb 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -12,15 +12,15 @@ import java.util.List; -public class KuduAsyncSideInfo extends BaseSideInfo { +public class KuduAsyncSideInfo extends SideInfo { - public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } @Override diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index 62d215d87..c03c470a7 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -9,9 +9,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; -public class KuduSideParser extends AbstractSideTableParser { +public class KuduSideParser extends AbsSideTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -52,7 +52,7 @@ public class KuduSideParser extends AbstractSideTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduSideTableInfo kuduSideTableInfo = new KuduSideTableInfo(); kuduSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduSideTableInfo); @@ -107,7 +107,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; - default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java index eb8df5ad9..c527ec268 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java @@ -1,9 +1,9 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; -public class KuduSideTableInfo extends AbstractSideTableInfo { +public class KuduSideTableInfo extends SideTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index dfc02dee6..b750e2d4a 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -191,7 +191,6 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info return KuduPredicate.newIsNotNullPredicate(column); case "IS NULL": return KuduPredicate.newIsNullPredicate(column); - default: } return null; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 4543b9618..99d19cdef 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,11 +18,13 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.kudu.client.AsyncKuduClient; import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; @@ -42,7 +44,7 @@ * @author gituser * @modify xiuzhu */ -public class KuduOutputFormat extends AbstractDtRichOutputFormat { +public class KuduOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; @@ -123,7 +125,7 @@ public void writeRecord(Tuple2 record) throws IOException { Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { - LOG.error("record insert failed ..{}", row.toString()); + LOG.error("record insert failed:{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 4c7c3bea8..0c90ffe94 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -3,7 +3,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kudu.table.KuduTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +39,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; @Override - public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public KuduSink genStreamSink(TargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; this.kuduMasters = kuduTableInfo.getKuduMasters(); this.tableName = kuduTableInfo.getTableName(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index 20302d44f..ea61bd325 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,8 +1,8 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -10,9 +10,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; -public class KuduSinkParser extends AbstractTableParser { +public class KuduSinkParser extends AbsTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -27,7 +27,7 @@ public class KuduSinkParser extends AbstractTableParser { public static final String SOCKET_READ_TIMEOUT_MS = "defaultSocketReadTimeoutMs"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduTableInfo kuduTableInfo = new KuduTableInfo(); kuduTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduTableInfo); @@ -86,7 +86,6 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; - default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java index 80e47761b..c086c7298 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; -public class KuduTableInfo extends AbstractTargetTableInfo { +public class KuduTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 8f0090db5..14cd847b5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,11 +21,11 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; -import com.esotericsoftware.minlog.Log; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -89,7 +89,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString("fs.hdfs.hadoopconf", yarnConfDir); + config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -109,7 +109,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - Log.info("applicationId={}", applicationId.toString()); + System.out.println("applicationId=" + applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); @@ -166,7 +166,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!"application".equals(it.next())) { + if (!(it.next()).equals("application")) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 0e5089bc9..3de1cdbc5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; -import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -80,7 +79,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - Log.info("----init yarn success ----"); + System.out.println("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index ca2d5bdd7..29cc4890a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -73,6 +73,7 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); + System.out.println(tips); LOG.info(tips); return applicationId; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 3373f3667..f90cfd973 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,17 +18,25 @@ package com.dtstack.flink.sql.side.mongo; -import com.dtstack.flink.sql.side.BaseAllReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; @@ -36,19 +44,13 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -60,7 +62,7 @@ * * @author xuqianjin */ -public class MongoAllReqRow extends BaseAllReqRow { +public class MongoAllReqRow extends AllReqRow { private static final long serialVersionUID = -675332795591842778L; @@ -76,7 +78,7 @@ public class MongoAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MongoAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -179,13 +181,34 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String database, String tableName) { + private MongoCollection getConn(String address, String userName, String password, String database, String tableName) { MongoCollection dbCollection; - mongoClient = new MongoClient(new MongoClientURI(address)); - db = mongoClient.getDatabase(database); - dbCollection = db.getCollection(tableName, Document.class); - return dbCollection; - + try { + MongoCredential credential; + String[] servers = StringUtils.split(address, ","); + String host; + Integer port; + String[] hostAndPort; + List lists = new ArrayList<>(); + for (String server : servers) { + hostAndPort = StringUtils.split(server, ":"); + host = hostAndPort[0]; + port = Integer.parseInt(hostAndPort[1]); + lists.add(new ServerAddress(host, port)); + } + if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { + credential = MongoCredential.createCredential(userName, database, password.toCharArray()); + // To connect to mongodb server + mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); + } else { + mongoClient = new MongoClient(lists); + } + db = mongoClient.getDatabase(database); + dbCollection = db.getCollection(tableName, Document.class); + return dbCollection; + } catch (Exception e) { + throw new RuntimeException("[connMongoDB]:" + e.getMessage()); + } } private void loadData(Map>> tmpCache) throws SQLException { @@ -195,7 +218,8 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), + tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index 4a33f1a69..b2d5c2686 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -36,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAllSideInfo extends BaseSideInfo { +public class MongoAllSideInfo extends SideInfo{ private static final long serialVersionUID = -1696860430075523841L; - public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index d85a9990d..42749e935 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -27,35 +27,33 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.mongodb.BasicDBObject; import com.mongodb.Block; -import com.mongodb.ConnectionString; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; -import com.mongodb.MongoClientSettings; +import com.mongodb.async.client.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; +import com.mongodb.connection.ClusterSettings; +import com.mongodb.connection.ConnectionPoolSettings; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -68,43 +66,68 @@ * * @author xuqianjin */ -public class MongoAsyncReqRow extends BaseAsyncReqRow { +public class MongoAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = -1183158242862673706L; private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); + private final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = 20; + private transient MongoClient mongoClient; private MongoDatabase db; - private MongoSideTableInfo mongoSideTableInfo; + private MongoSideTableInfo MongoSideTableInfo; - public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - mongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); - connMongoDb(); + MongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); + connMongoDB(); } - public void connMongoDb() throws Exception { - String address = mongoSideTableInfo.getAddress(); - ConnectionString connectionString = new ConnectionString(address); - - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(connectionString) + public void connMongoDB() throws Exception { + MongoCredential mongoCredential; + String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); + String host; + Integer port; + String[] hostAndPort; + List lists = new ArrayList<>(); + for (String server : servers) { + hostAndPort = StringUtils.split(server, ":"); + host = hostAndPort[0]; + port = Integer.parseInt(hostAndPort[1]); + lists.add(new ServerAddress(host, port)); + } + ClusterSettings clusterSettings = ClusterSettings.builder().hosts(lists).build(); + ConnectionPoolSettings connectionPoolSettings = ConnectionPoolSettings.builder() + .maxSize(DEFAULT_MAX_DB_CONN_POOL_SIZE) .build(); - mongoClient = MongoClients.create(settings); - db = mongoClient.getDatabase(mongoSideTableInfo.getDatabase()); + if (!StringUtils.isEmpty(MongoSideTableInfo.getUserName()) || !StringUtils.isEmpty(MongoSideTableInfo.getPassword())) { + mongoCredential = MongoCredential.createCredential(MongoSideTableInfo.getUserName(), MongoSideTableInfo.getDatabase(), + MongoSideTableInfo.getPassword().toCharArray()); + MongoClientSettings settings = MongoClientSettings.builder().credential(mongoCredential) + .clusterSettings(clusterSettings) + .connectionPoolSettings(connectionPoolSettings) + .build(); + mongoClient = MongoClients.create(settings); + } else { + MongoClientSettings settings = MongoClientSettings.builder().clusterSettings(clusterSettings) + .connectionPoolSettings(connectionPoolSettings) + .build(); + mongoClient = MongoClients.create(settings); + } + db = mongoClient.getDatabase(MongoSideTableInfo.getDatabase()); } @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - BasicDBObject basicDbObject = new BasicDBObject(); + BasicDBObject basicDBObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputCopy.row().getField(conValIndex); @@ -112,14 +135,14 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce dealMissKey(inputCopy, resultFuture); return; } - basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); + basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); } try { // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); if (null != filterCondition) { - basicDbObject.append(info.getFieldName(), filterCondition); + basicDBObject.append(info.getFieldName(), filterCondition); } return info; }).count(); @@ -127,7 +150,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDbObject.values()); + String key = buildCacheKey(basicDBObject.values()); if (openCache()) { CacheObj val = getFromCache(key); if (val != null) { @@ -149,7 +172,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } AtomicInteger atomicInteger = new AtomicInteger(0); - MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); + MongoCollection dbCollection = db.getCollection(MongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); Block printDocumentBlock = new Block() { @Override @@ -175,7 +198,7 @@ public void onResult(final Void result, final Throwable t) { } } }; - dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); + dbCollection.find(basicDBObject).forEach(printDocumentBlock, callbackWhenFinished); } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index 19534d4e1..de81ed1f1 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -20,13 +20,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Lists; import java.util.List; @@ -36,16 +39,16 @@ * * @author xuqianjin */ -public class MongoAsyncSideInfo extends BaseSideInfo { +public class MongoAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -3694857194254465989L; - public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 4fe1ebee1..5edc12419 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -19,13 +19,15 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +35,7 @@ * * @author xuqianjin */ -public class MongoSideParser extends AbstractSideTableParser { +public class MongoSideParser extends AbsSideTableParser { public static final String ADDRESS_KEY = "address"; @@ -46,7 +48,7 @@ public class MongoSideParser extends AbstractSideTableParser { public static final String DATABASE_KEY = "database"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoSideTableInfo mongoSideTableInfo = new MongoSideTableInfo(); mongoSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoSideTableInfo); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index c83de7bbc..a5c834469 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class MongoSideTableInfo extends AbstractSideTableInfo { +public class MongoSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index cbcba8f97..2f1a2092d 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -55,7 +55,6 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { return new BasicDBObject("$exists", true); case "IS NULL": return new BasicDBObject("$exists", false); - default: } return null; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 3f1f9f7e2..337fbad59 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,16 +19,20 @@ package com.dtstack.flink.sql.sink.mongo; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.result.UpdateResult; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.result.UpdateResult; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.types.ObjectId; import org.slf4j.Logger; @@ -44,7 +48,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends AbstractDtRichOutputFormat { +public class MongoOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; @@ -121,8 +125,30 @@ public void close() { } private void establishConnection() { - mongoClient = new MongoClient(new MongoClientURI(address)); - db = mongoClient.getDatabase(database); + try { + MongoCredential credential; + String[] servers = StringUtils.split(address, ","); + String host; + Integer port; + String[] hostAndPort; + List lists = new ArrayList<>(); + for (String server : servers) { + hostAndPort = StringUtils.split(server, ":"); + host = hostAndPort[0]; + port = Integer.parseInt(hostAndPort[1]); + lists.add(new ServerAddress(host, port)); + } + if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { + credential = MongoCredential.createCredential(userName, database, password.toCharArray()); + // To connect to mongodb server + mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); + } else { + mongoClient = new MongoClient(lists); + } + db = mongoClient.getDatabase(database); + } catch (Exception e) { + throw new IllegalArgumentException("[connMongoDB]:" + e.getMessage()); + } } private MongoOutputFormat() { diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 3f34055ec..4e28d8fd2 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.mongo.table.MongoTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -54,7 +54,7 @@ public MongoSink() { } @Override - public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public MongoSink genStreamSink(TargetTableInfo targetTableInfo) { MongoTableInfo mongoTableInfo = (MongoTableInfo) targetTableInfo; this.address = mongoTableInfo.getAddress(); this.tableName = mongoTableInfo.getTableName(); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java index f093b70aa..d4810fb6d 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; /** * Reason: @@ -34,7 +34,7 @@ * @author xuqianjin */ -public class MongoSinkParser extends AbstractTableParser { +public class MongoSinkParser extends AbsTableParser { private static final String CURR_TYPE = "mongo"; @@ -49,7 +49,7 @@ public class MongoSinkParser extends AbstractTableParser { public static final String DATABASE_KEY = "database"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoTableInfo mongoTableInfo = new MongoTableInfo(); mongoTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoTableInfo); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index c9d94dfe6..02a96d6bb 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; /** @@ -30,7 +30,7 @@ */ -public class MongoTableInfo extends AbstractTargetTableInfo { +public class MongoTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "mongo"; diff --git a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java new file mode 100644 index 000000000..33a0233ac --- /dev/null +++ b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flinkx; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index eb294e64e..b6b7f45cb 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author xuchao */ -public class MysqlAllReqRow extends AbstractRdbAllReqRow { +public class MysqlAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(MYSQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 503dbf9a3..07b06ba38 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class MysqlAllSideInfo extends RdbAllSideInfo { - public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index d52f1918f..6120767ef 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,6 +30,8 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -42,9 +44,12 @@ */ public class MysqlAsyncReqRow extends RdbAsyncReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(MysqlAsyncReqRow.class); + private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -56,7 +61,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", MYSQL_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -68,10 +73,10 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index 9555897aa..5d6ed370d 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { - public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index b0a38e344..40f68e7e4 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class MysqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "mysql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 0a1749f04..8c1146ede 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; @@ -33,7 +33,7 @@ * @author xuchao */ -public class MysqlSink extends AbstractRdbSink implements IStreamSinkGener { +public class MysqlSink extends RdbSink implements IStreamSinkGener { public MysqlSink() { super(new MySQLDialect()); @@ -42,7 +42,7 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 49105a7a8..2247eb8cc 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mysql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class MysqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "mysql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java new file mode 100644 index 000000000..33a0233ac --- /dev/null +++ b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.dtstack.flinkx; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 3be6687ec..18d9ba045 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -36,22 +36,22 @@ /** * side operator with cache for all(period reload) */ -public class OracleAllReqRow extends AbstractRdbAllReqRow { +public class OracleAllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(OracleAllReqRow.class); private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(ORACLE_DRIVER); Map addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 342533681..361366929 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -19,17 +19,18 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; public class OracleAllSideInfo extends RdbAllSideInfo { - public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 938fd870c..f973ffee2 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,14 +30,19 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; public class OracleAsyncReqRow extends RdbAsyncReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(OracleAsyncReqRow.class); + private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -48,7 +53,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -60,9 +65,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 658020b1c..8e801970f 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -20,20 +20,19 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; - import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -56,7 +55,7 @@ public String wrapperPlaceholder(String fieldName) { String rpadFormat = "rpad(?, %d, ' ')"; if (StringUtils.contains(type.toLowerCase(), "char")) { - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(rpadFormat, charLength); diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index 897b77d30..f9124bd10 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.oracle.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -27,8 +27,8 @@ public class OracleSideParser extends RdbSideParser { private static final String CURR_TYPE = "oracle"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index 3a320658c..a687aa012 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; @@ -41,7 +41,7 @@ public class OracleDialect implements JDBCDialect { private List fieldList; private List fieldTypeList; - private List fieldExtraInfoList; + private List fieldExtraInfoList; @Override public boolean canHandle(String url) { @@ -133,7 +133,7 @@ public String wrapperPlaceholder(String fieldName) { String type = fieldTypeList.get(pos); if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(RPAD_FORMAT, charLength); @@ -151,7 +151,7 @@ public void setFieldTypeList(List fieldTypeList) { this.fieldTypeList = fieldTypeList; } - public void setFieldExtraInfoList(List fieldExtraInfoList) { + public void setFieldExtraInfoList(List fieldExtraInfoList) { this.fieldExtraInfoList = fieldExtraInfoList; } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index f916b55f2..7a37aa6cd 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class OracleSink extends AbstractRdbSink implements IStreamSinkGener { +public class OracleSink extends RdbSink implements IStreamSinkGener { public OracleSink() { super(new OracleDialect()); @@ -42,7 +42,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { ((OracleDialect) jdbcDialect).setFieldExtraInfoList(fieldExtraInfoList); JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index 1732c8cc2..aff096bd3 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.oracle.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class OracleSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "oracle"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index 5fbf0824b..f9e0950dd 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -1,26 +1,9 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -32,12 +15,7 @@ import java.util.List; import java.util.Map; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ -public class PolardbAllReqRow extends AbstractRdbAllReqRow { +public class PolardbAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -45,18 +23,18 @@ public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PolardbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(POLARDB_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index c46c9dfd6..822cbaa70 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -1,37 +1,15 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbAllSideInfo extends RdbAllSideInfo { - public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 13d25b111..3cbb0b368 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -1,25 +1,8 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -33,18 +16,13 @@ import java.util.List; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(PolardbAsyncReqRow.class); private final static String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PolardbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -55,7 +33,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); mysqlClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POLARDB_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -67,9 +45,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index 15237956b..bad71375f 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -1,38 +1,16 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { - public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 71aad7cbb..348b3aed9 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -1,39 +1,16 @@ - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index c69f31e8f..44d558140 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,33 +1,11 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.dtstack.flink.sql.sink.polardb; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ -public class PolardbSink extends AbstractRdbSink { + +public class PolardbSink extends RdbSink { public PolardbSink() { super(new PolardbDialect()); } @@ -35,7 +13,7 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index b4f02665e..68eead938 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -1,38 +1,16 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; -/** - * Date: 2019/12/20 - * Company: www.dtstack.com - * @author yinxi - */ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; } diff --git a/pom.xml b/pom.xml index de963240c..2794f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,6 @@ mysql hbase elasticsearch5 - elasticsearch6 mongo redis5 launcher @@ -34,6 +33,7 @@ impala db2 polardb + elasticsearch6 @@ -54,20 +54,6 @@ - - org.jacoco - jacoco-maven-plugin - 0.7.8 - - - - prepare-agent - report - - - - - pl.project13.maven git-commit-id-plugin diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 0ec8f96ce..6d68cfdca 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author tcm */ -public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { +public class PostgresqlAllReqRow extends RdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(POSTGRESQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index a4a52eabc..d383ee46d 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 42aa39a51..86fed91fa 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -60,7 +60,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); pgClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", POSTGRESQL_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) @@ -69,9 +69,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index f19a488bc..1d89f4894 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index be0c5ea8b..faee2c704 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.postgresql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class PostgresqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "postgresql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index bb731152c..cf45a6bd9 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * @author maqi */ -public class PostgresqlSink extends AbstractRdbSink implements IStreamSinkGener { +public class PostgresqlSink extends RdbSink implements IStreamSinkGener { public PostgresqlSink() { super(new PostgresqlDialect()); } @@ -36,7 +36,7 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index e94cfc8a0..5943b5e52 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -35,8 +35,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "postgresql"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java similarity index 92% rename from rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java rename to rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 2e9986742..60e3d817f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -19,15 +19,13 @@ package com.dtstack.flink.sql.side.rdb.all; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.BaseAllReqRow; -import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.google.common.collect.Lists; @@ -58,11 +56,11 @@ * @author maqi */ -public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { +public abstract class RdbAllReqRow extends AllReqRow { private static final long serialVersionUID = 2098635140857937718L; - private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbAllReqRow.class); + private static final Logger LOG = LoggerFactory.getLogger(RdbAllReqRow.class); private static final int CONN_RETRY_NUM = 3; @@ -70,18 +68,10 @@ public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { + public RdbAllReqRow(SideInfo sideInfo) { super(sideInfo); } - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("rdb dim table config info: {} ", tableInfo.toString()); - } - - @Override protected void initCache() throws SQLException { Map>> newCache = Maps.newConcurrentMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 143017133..3b7444b77 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -23,16 +23,14 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -46,21 +44,20 @@ * @author maqi */ -public class RdbAllSideInfo extends BaseSideInfo { +public class RdbAllSideInfo extends SideInfo { private static final long serialVersionUID = -5858335638589472159L; - private static final Logger LOG = LoggerFactory.getLogger(RdbAllSideInfo.class.getSimpleName()); - public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), sideTableInfo.getPredicateInfoes()); - LOG.info("--------dimension sql query-------\n{}" + sqlCondition); + System.out.println("--------dimension sql query-------\n" + sqlCondition); } public String getAdditionalWhereClause() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2ba34ad6b..2d767ccd9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,11 +20,8 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.DateUtil; import io.vertx.core.json.JsonArray; @@ -32,7 +29,6 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; -import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -53,7 +49,7 @@ * @author maqi */ -public class RdbAsyncReqRow extends BaseAsyncReqRow { +public class RdbAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; @@ -63,9 +59,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - - public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; + public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; @@ -75,25 +69,10 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSqlClient; + private transient SQLClient rdbSQLClient; - public RdbAsyncReqRow(BaseSideInfo sideInfo) { + public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); - init(sideInfo); - } - - protected void init(BaseSideInfo sideInfo) { - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); - int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; - rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - LOG.info("rdb dim table config info: {} ", rdbSideTableInfo.toString()); } @Override @@ -130,7 +109,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - rdbSqlClient.getConnection(conn -> { + rdbSQLClient.getConnection(conn -> { if (conn.failed()) { //Treatment failures resultFuture.completeExceptionally(conn.cause()); @@ -197,9 +176,9 @@ private Object convertDataType(Object val) { } else if (val instanceof Instant) { } else if (val instanceof Timestamp) { - val = DateUtil.timestampToString((Timestamp) val); + val = DateUtil.getStringFromTimestamp((Timestamp) val); } else if (val instanceof java.util.Date) { - val = DateUtil.dateToString((java.util.Date)val); + val = DateUtil.getStringFromDate((java.sql.Date) val); } else { val = val.toString(); } @@ -248,8 +227,8 @@ public Row fillData(Row input, Object line) { @Override public void close() throws Exception { super.close(); - if (rdbSqlClient != null) { - rdbSqlClient.close(); + if (rdbSQLClient != null) { + rdbSQLClient.close(); } } @@ -264,8 +243,8 @@ public String buildCacheKey(JsonArray jsonArray) { return sb.toString(); } - public void setRdbSqlClient(SQLClient rdbSqlClient) { - this.rdbSqlClient = rdbSqlClient; + public void setRdbSQLClient(SQLClient rdbSQLClient) { + this.rdbSQLClient = rdbSQLClient; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 9cbd43989..9e8c13080 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -23,8 +23,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; @@ -33,8 +33,6 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -49,18 +47,16 @@ * @author maqi */ -public class RdbAsyncSideInfo extends BaseSideInfo { +public class RdbAsyncSideInfo extends SideInfo { private static final long serialVersionUID = 1942629132469918611L; - private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncSideInfo.class); - - public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -78,7 +74,7 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), equalFieldList, sqlJoinCompareOperate, sideTableInfo.getPredicateInfoes()); - LOG.info("----------dimension sql query-----------\n{}", sqlCondition); + System.out.println("----------dimension sql query-----------\n" + sqlCondition); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java index 542258257..fcca4f0ef 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java @@ -25,19 +25,13 @@ import javax.sql.DataSource; import java.beans.PropertyVetoException; import java.sql.SQLException; -/** - * Date: 2019/9/17 - * Company: www.dtstack.com - * @author maqi - */ + public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { @Override public DataSource getDataSource(JsonObject config) throws SQLException { String url = config.getString("url"); - if (url == null) { - throw new NullPointerException("url cannot be null"); - } + if (url == null) throw new NullPointerException("url cannot be null"); String driverClass = config.getString("driver_class"); String user = config.getString("user"); String password = config.getString("password"); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index 79942ae11..b9811b0ee 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,10 +33,10 @@ * @author maqi */ -public class RdbSideParser extends AbstractSideTableParser { +public class RdbSideParser extends AbsSideTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbSideTableInfo rdbTableInfo = new RdbSideTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 43d0a6ee8..2c8c18fcd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class RdbSideTableInfo extends AbstractSideTableInfo { +public class RdbSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; @@ -101,15 +101,4 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } - - @Override - public String toString() { - String cacheInfo = super.toString(); - String connectionInfo = "Rdb Side Connection Info{" + - "url='" + url + '\'' + - ", tableName='" + tableName + '\'' + - ", schema='" + schema + '\'' + - '}'; - return cacheInfo + " , " + connectionInfo; - } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 49d163d4d..1205c5521 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,7 +78,6 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); - default: } return obj; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index f476bfadf..e9a4f391f 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -29,7 +29,7 @@ public class JDBCOptions { - private String dbUrl; + private String dbURL; private String tableName; private String driverName; private String username; @@ -37,9 +37,9 @@ public class JDBCOptions { private String schema; private JDBCDialect dialect; - private JDBCOptions(String dbUrl, String tableName, String driverName, String username, + private JDBCOptions(String dbURL, String tableName, String driverName, String username, String password, String schema, JDBCDialect dialect) { - this.dbUrl = dbUrl; + this.dbURL = dbURL; this.tableName = tableName; this.driverName = driverName; this.username = username; @@ -48,8 +48,8 @@ private JDBCOptions(String dbUrl, String tableName, String driverName, String us this.dialect = dialect; } - public String getDbUrl() { - return dbUrl; + public String getDbURL() { + return dbURL; } public String getTableName() { @@ -84,7 +84,7 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JDBCOptions) { JDBCOptions options = (JDBCOptions) o; - return Objects.equals(dbUrl, options.dbUrl) && + return Objects.equals(dbURL, options.dbURL) && Objects.equals(tableName, options.tableName) && Objects.equals(driverName, options.driverName) && Objects.equals(username, options.username) && @@ -100,7 +100,7 @@ public boolean equals(Object o) { * Builder of {@link JDBCOptions}. */ public static class Builder { - private String dbUrl; + private String dbURL; private String tableName; private String driverName; private String username; @@ -152,8 +152,8 @@ public Builder setSchema(String schema) { /** * required, JDBC DB url. */ - public Builder setDbUrl(String dbUrl) { - this.dbUrl = dbUrl; + public Builder setDBUrl(String dbURL) { + this.dbURL = dbURL; return this; } @@ -163,7 +163,7 @@ public Builder setDialect(JDBCDialect dialect) { } public JDBCOptions build() { - checkNotNull(dbUrl, "No dbURL supplied."); + checkNotNull(dbURL, "No dbURL supplied."); checkNotNull(tableName, "No tableName supplied."); if (this.driverName == null) { @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbUrl, tableName, driverName, username, password, schema, dialect); + return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); } } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java similarity index 90% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java index ee51c22d1..84eea68ff 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -44,8 +44,11 @@ * * @author maqi */ -public abstract class AbstractRdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - protected String dbUrl; +public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + + protected String driverName; + + protected String dbURL; protected String userName; @@ -83,20 +86,20 @@ public abstract class AbstractRdbSink implements RetractStreamTableSink, Se public List fieldList; public List fieldTypeList; - public List fieldExtraInfoList; + public List fieldExtraInfoList; - public AbstractRdbSink(JDBCDialect jdbcDialect) { + public RdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbUrl = rdbTableInfo.getUrl(); + this.dbURL = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 2a3ce5e90..73f17d8fd 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; @@ -36,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFormat { +public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; @@ -47,16 +47,16 @@ public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFo protected final String username; protected final String password; - private final String driverName; + private final String drivername; protected final String dbURL; protected transient Connection connection; - public AbstractJDBCOutputFormat(String username, String password, String driverName, String dbUrl) { + public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { this.username = username; this.password = password; - this.driverName = driverName; - this.dbURL = dbUrl; + this.drivername = drivername; + this.dbURL = dbURL; } @Override @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(driverName, getClass().getClassLoader()); + JDBCUtils.forName(drivername, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index e8b6dc8a4..e1b2cc6f4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -25,7 +25,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; +import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -44,7 +44,6 @@ /** * An upsert OutputFormat for JDBC. - * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -84,7 +83,7 @@ public JDBCUpsertOutputFormat( long flushIntervalMills, boolean allReplace, String updateMode) { - super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbUrl()); + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); this.schema = options.getSchema(); this.tableName = options.getTableName(); this.dialect = options.getDialect(); @@ -112,11 +111,12 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); - LOG.info("execute insert sql： {}", insertSql); - jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); + String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); + LOG.info("execute insert sql： {}", insertSQL); + System.out.println("execute insert sql :" + insertSQL); + jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); } else { - jdbcWriter = AbstractUpsertWriter.create( + jdbcWriter = UpsertWriter.create( dialect, schema, tableName, fieldNames, fieldTypes, keyFields, partitionFields, getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index aca7acb6c..9bbff3b4d 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,8 +17,9 @@ */ package com.dtstack.flink.sql.sink.rdb.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +31,9 @@ * * @author maqi */ -public class RdbSinkParser extends AbstractTableParser { +public class RdbSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbTableInfo rdbTableInfo = new RdbTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index ccdf28e48..e490978e4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.rdb.table; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; + /** * Reason: * Date: 2018/11/27 @@ -29,7 +31,7 @@ * * @author maqi */ -public class RdbTableInfo extends AbstractTargetTableInfo { +public class RdbTableInfo extends TargetTableInfo { public static final String URL_KEY = "url"; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 3559d4376..ca2bd4c66 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -34,7 +34,6 @@ /** * Just append record to jdbc, can not receive retract/delete message. - * @author maqi */ public class AppendOnlyWriter implements JDBCWriter { @@ -42,16 +41,16 @@ public class AppendOnlyWriter implements JDBCWriter { private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); - private final String insertSql; + private final String insertSQL; private final int[] fieldTypes; private transient PreparedStatement statement; private transient List rows; // only use metric - private transient AbstractDtRichOutputFormat metricOutputFormat; + private transient DtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSql, int[] fieldTypes, AbstractDtRichOutputFormat metricOutputFormat) { - this.insertSql = insertSql; + public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { + this.insertSQL = insertSQL; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; } @@ -64,7 +63,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.statement = connection.prepareStatement(insertSql); + this.statement = connection.prepareStatement(insertSQL); } /** diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java similarity index 84% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java index 2c070b680..ce3a1c680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -41,15 +41,14 @@ /** * Upsert writer to deal with upsert, delete message.dd - * @author maqi */ -public abstract class AbstractUpsertWriter implements JDBCWriter { +public abstract class UpsertWriter implements JDBCWriter { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(AbstractUpsertWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); - public static AbstractUpsertWriter create( + public static UpsertWriter create( JDBCDialect dialect, String schema, String tableName, @@ -59,7 +58,7 @@ public static AbstractUpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - AbstractDtRichOutputFormat metricOutputFormat) { + DtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -68,18 +67,20 @@ public static AbstractUpsertWriter create( int[] pkTypes = fieldTypes == null ? null : Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSql); + String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSQL); + System.out.println("deleteSQL is :" + deleteSQL); - Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); - LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); + Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); + System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); - return upsertSql.map((Function) sql -> + return upsertSQL.map((Function) sql -> new UpsertWriterUsingUpsertStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, sql, metricOutputFormat)) + fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) .orElseGet(() -> new UpsertWriterUsingInsertUpdateStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, + fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, dialect.getRowExistsStatement(tableName, keyFields), dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), dialect.getUpdateStatement(tableName, fieldNames, keyFields), @@ -89,19 +90,19 @@ public static AbstractUpsertWriter create( final int[] fieldTypes; final int[] pkTypes; private final int[] pkFields; - private final String deleteSql; + private final String deleteSQL; private final boolean objectReuse; private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient AbstractDtRichOutputFormat metricOutputFormat; + private transient DtRichOutputFormat metricOutputFormat; - private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, AbstractDtRichOutputFormat metricOutputFormat) { + private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; - this.deleteSql = deleteSql; + this.deleteSQL = deleteSQL; this.objectReuse = objectReuse; this.metricOutputFormat = metricOutputFormat; } @@ -114,7 +115,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.deleteStatement = connection.prepareStatement(deleteSql); + this.deleteStatement = connection.prepareStatement(deleteSQL); } @Override @@ -169,6 +170,7 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { + System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); @@ -205,10 +207,10 @@ private Row getPrimaryKey(Row row) { // ---------------------------------------------------------------------------------------- - private static final class UpsertWriterUsingUpsertStatement extends AbstractUpsertWriter { + private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { private static final long serialVersionUID = 1L; - private final String upsertSql; + private final String upsertSQL; private transient PreparedStatement upsertStatement; @@ -217,11 +219,11 @@ private UpsertWriterUsingUpsertStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSql, - String upsertSql, - AbstractDtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); - this.upsertSql = upsertSql; + String deleteSQL, + String upsertSQL, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); + this.upsertSQL = upsertSQL; } @Override @@ -232,7 +234,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - upsertStatement = connection.prepareStatement(upsertSql); + upsertStatement = connection.prepareStatement(upsertSQL); } @Override @@ -262,12 +264,12 @@ public void close() throws SQLException { } } - private static final class UpsertWriterUsingInsertUpdateStatement extends AbstractUpsertWriter { + private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { private static final long serialVersionUID = 1L; - private final String existSql; - private final String insertSql; - private final String updateSql; + private final String existSQL; + private final String insertSQL; + private final String updateSQL; private transient PreparedStatement existStatement; private transient PreparedStatement insertStatement; @@ -278,15 +280,15 @@ private UpsertWriterUsingInsertUpdateStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSql, - String existSql, - String insertSql, - String updateSql, - AbstractDtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); - this.existSql = existSql; - this.insertSql = insertSql; - this.updateSql = updateSql; + String deleteSQL, + String existSQL, + String insertSQL, + String updateSQL, + DtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); + this.existSQL = existSQL; + this.insertSQL = insertSQL; + this.updateSQL = updateSQL; } @Override @@ -297,9 +299,9 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - existStatement = connection.prepareStatement(existSql); - insertStatement = connection.prepareStatement(insertSql); - updateStatement = connection.prepareStatement(updateSql); + existStatement = connection.prepareStatement(existSQL); + insertStatement = connection.prepareStatement(insertSQL); + updateStatement = connection.prepareStatement(updateSQL); } @Override diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 5d09c8f3d..0d257c5b9 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -18,10 +18,6 @@ package com.dtstack.flink.sql.side.redis; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAllReqRow; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -33,34 +29,22 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.JedisCommands; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisSentinelPool; +import redis.clients.jedis.*; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; - -import java.util.Calendar; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; -/** - * @author yanxi - */ -public class RedisAllReqRow extends BaseAllReqRow { +import java.util.stream.Collectors; + +public class RedisAllReqRow extends AllReqRow{ private static final long serialVersionUID = 7578879189085344807L; @@ -78,7 +62,7 @@ public class RedisAllReqRow extends BaseAllReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.redisSideReqRow = new RedisSideReqRow(super.sideInfo); } @@ -106,7 +90,7 @@ protected void reloadCache() { } cacheRef.set(newCache); - LOG.info("----- Redis all cacheRef reload end:{}", Calendar.getInstance()); + LOG.info("----- Redis all cacheRef reload end:{}", newCache.size()); } @Override diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index ec0fa0d15..81e7b1f4f 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -20,27 +20,25 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; -/** - * @author yanxi - */ -public class RedisAllSideInfo extends BaseSideInfo { + +public class RedisAllSideInfo extends SideInfo { private static final long serialVersionUID = 1998703966487857613L; - public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d53bbed8c..f2e164922 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,18 +18,8 @@ package com.dtstack.flink.sql.side.redis; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.CacheMissVal; -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -39,20 +29,24 @@ import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; +import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Consumer; -/** - * @author yanxi - */ -public class RedisAsyncReqRow extends BaseAsyncReqRow { + +public class RedisAsyncReqRow extends AsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -70,7 +64,7 @@ public class RedisAsyncReqRow extends BaseAsyncReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index c23035ec7..9bdefe8c5 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -20,26 +20,24 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.BaseSideInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; -/** - * @author yanxi - */ -public class RedisAsyncSideInfo extends BaseSideInfo { + +public class RedisAsyncSideInfo extends SideInfo { private static final long serialVersionUID = -4851348392924455039L; - public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index 514984f0a..f8a83b868 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -18,19 +18,16 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.table.AbstractSideTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSideTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -/** - * @author yanxi - */ -public class RedisSideParser extends AbstractSideTableParser { +public class RedisSideParser extends AbsSideTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisSideTableInfo redisSideTableInfo = new RedisSideTableInfo(); redisSideTableInfo.setName(tableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 17557f916..5007868e9 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.ISideReqRow; -import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.SideInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,9 +40,9 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; - private BaseSideInfo sideInfo; + private SideInfo sideInfo; - public RedisSideReqRow(BaseSideInfo sideInfo){ + public RedisSideReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -72,7 +72,7 @@ public Row fillData(Row input, Object sideInput) { return row; } - public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 5224134c8..88cfcb8a2 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.google.common.base.Preconditions; -/** - * @author yanxi - */ -public class RedisSideTableInfo extends AbstractSideTableInfo { + +public class RedisSideTableInfo extends SideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ae4fe5a4b..0bd053473 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,13 +18,14 @@ package com.dtstack.flink.sql.sink.redis; -import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; + +import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; @@ -41,10 +42,7 @@ import java.util.List; import java.util.Set; -/** - * @author yanxi - */ -public class RedisOutputFormat extends AbstractDtRichOutputFormat { +public class RedisOutputFormat extends DtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; @@ -142,8 +140,7 @@ private void establishConnection() { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); - default: + jedis = new JedisCluster(addresses, timeout, timeout,10, password, poolConfig); } } @@ -159,10 +156,10 @@ public void writeRecord(Tuple2 record) throws IOException { return; } - HashMap map = new HashMap<>(8); - for (String primaryKey : primaryKeys) { - for (int i = 0; i < fieldNames.length; i++) { - if (fieldNames[i].equals(primaryKey)) { + HashMap map = new HashMap<>(); + for (String primaryKey : primaryKeys){ + for (int i=0; i kvList = new LinkedList<>(); for (String primaryKey : primaryKeys){ - StringBuilder primaryKv = new StringBuilder(); + StringBuilder primaryKV = new StringBuilder(); int index = map.get(primaryKey).intValue(); - primaryKv.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKv.toString()); + primaryKV.append(primaryKey).append(":").append(row.getField(index)); + kvList.add(primaryKV.toString()); } String perKey = String.join(":", kvList); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index cc49a3ba8..d2e28c01f 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.redis.table.RedisTableInfo; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,9 +33,7 @@ import org.apache.flink.types.Row; import java.util.List; -/** - * @author yanxi - */ + public class RedisSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -69,7 +67,7 @@ public RedisSink(){ } @Override - public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + public RedisSink genStreamSink(TargetTableInfo targetTableInfo) { RedisTableInfo redisTableInfo = (RedisTableInfo) targetTableInfo; this.url = redisTableInfo.getUrl(); this.database = redisTableInfo.getDatabase(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index e965eeecb..0c9b0a64a 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbstractTableParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -27,12 +27,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -/** - * @author yanxi - */ -public class RedisSinkParser extends AbstractTableParser { + +public class RedisSinkParser extends AbsTableParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisTableInfo redisTableInfo = new RedisTableInfo(); redisTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, redisTableInfo); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 2425f8c38..60d6dd12f 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbstractTargetTableInfo; +import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.base.Preconditions; -/** - * @author yanxi - */ -public class RedisTableInfo extends AbstractTargetTableInfo { + +public class RedisTableInfo extends TargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index b032fdce6..e46e0a76f 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.types.Row; @@ -67,7 +67,7 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { ServersocketSourceTableInfo tableInfo; public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, - Map rowAndFieldMapping, List fieldExtraInfos) { + Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); @@ -94,7 +94,7 @@ public void run(SourceContext ctx) throws Exception { while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { String record = buffer.substring(0, delimPos); // truncate trailing carriage return - if ("\n".equals(delimiter) && "\r".endsWith(record)) { + if (delimiter.equals("\n") && record.endsWith("\r")) { record = record.substring(0, record.length() - 1); } try { diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java index b4dda5f3b..3ab01fb42 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ */ public class ServersocketSource implements IStreamSourceGener
{ @Override - public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; String tableName = serversocketSourceTableInfo.getName(); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java index f2d861b17..b52e38e59 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java @@ -17,8 +17,8 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbstractSourceParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +30,9 @@ * * @author maqi */ -public class ServersocketSourceParser extends AbstractSourceParser { +public class ServersocketSourceParser extends AbsSourceParser { @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); serversocketSourceTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 3123b477f..796728eb3 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.google.common.base.Preconditions; /** @@ -27,8 +27,8 @@ * * @author maqi */ -public class ServersocketSourceTableInfo extends AbstractSourceTableInfo { - +public class ServersocketSourceTableInfo extends SourceTableInfo { + //version private static final String CURR_TYPE = "serversocket"; public static final String HOSTNAME_KEY = "host"; diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index dbae271e4..961539fde 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -20,43 +20,39 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.DriverManager; import java.util.List; import java.util.Map; - /** - * side operator with cache for all(period reload) - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi + * side operator with cache for all(period reload) */ -public class SqlserverAllReqRow extends AbstractRdbAllReqRow { +public class SqlserverAllReqRow extends RdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAllReqRow.class); private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbUrl, String userName, String password) { + public Connection getConn(String dbURL, String userName, String password) { try { Class.forName(SQLSERVER_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + //addParams.put("useCursorFetch", "true"); + String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index a40af5697..2b9913e7e 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -20,18 +20,14 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverAllSideInfo extends RdbAllSideInfo { - public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index a63d28607..cb4240ca6 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -33,18 +33,14 @@ import java.util.List; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAsyncReqRow.class); private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -55,7 +51,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) + .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -68,9 +64,9 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); + vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index e8a487721..347ca9bf4 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -20,21 +20,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; - import java.util.List; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { - public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index e06c13898..3631dcd8a 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -18,20 +18,16 @@ package com.dtstack.flink.sql.side.sqlserver.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; -/** - * Date: 2019/11/26 - * Company: www.dtstack.com - * @author maqi - */ + public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 3c331cdd0..dbbeb820e 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class SqlserverSink extends AbstractRdbSink implements IStreamSinkGener { +public class SqlserverSink extends RdbSink implements IStreamSinkGener { public SqlserverSink() { super(new SqlserverDialect()); @@ -38,7 +38,7 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(dbUrl) + .setDBUrl(dbURL) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index 5300884bc..a695d6c3b 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.sqlserver.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.TableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class SqlserverSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "sqlserver"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; } From 03a491311e6f56fb1eab2c6287e7ee2d2b140a15 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 8 Apr 2020 17:38:59 +0800 Subject: [PATCH 090/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dsonar=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 2 +- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 -- .../java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index b8530f224..56225cefb 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -121,7 +121,7 @@ public ReplaceInfo getReplaceInfo(String field){ private List makeFormula(String formula){ if(formula == null || formula.length() <= 0){ - Lists.newArrayList(); + return Lists.newArrayList(); } List result = Lists.newArrayList(); for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a3189b0a6..e89ec0b46 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -22,7 +22,6 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; @@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Set; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java index 89667ee28..7af51be3d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -130,7 +130,7 @@ public ReplaceInfo getReplaceInfo(String field){ private List makeFormula(String formula){ if(formula == null || formula.length() <= 0){ - Lists.newArrayList(); + return Lists.newArrayList(); } List result = Lists.newArrayList(); for(String meta: splitIgnoreQuotaBrackets(formula, "\\+")){ From a0e7d5b31e697b5162f13bc4f7efb537b8c93e35 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 8 Apr 2020 17:56:08 +0800 Subject: [PATCH 091/378] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E7=BC=BA=E5=B0=91=E7=9A=84=E4=BF=AE=E6=94=B9=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 823cd5143..3c9f37811 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -146,6 +146,8 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode()); StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); StreamQueryConfig streamQueryConfig = StreamEnvConfigManager.getStreamQueryConfig(tableEnv, paramsInfo.getConfProp()); + // init global flinkPlanner + FlinkPlanner.createFlinkPlanner(tableEnv.getFrameworkConfig(), tableEnv.getPlanner(), tableEnv.getTypeFactory()); SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); From 8aacd7e948c284d0ad15eec4e107ed7c4566ebfa Mon Sep 17 00:00:00 2001 From: WTZ468071157 Date: Thu, 9 Apr 2020 10:43:15 +0800 Subject: [PATCH 092/378] fix import * --- .../flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 5a1158125..bab9cc928 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -43,7 +43,11 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Calendar; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; From 5962508b3cb9010cec6d0f0a956adadc956cf23c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Apr 2020 10:58:54 +0800 Subject: [PATCH 093/378] fix import * --- .../dtstack/flink/sql/launcher/ClusterClientFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 1da0db7a3..391e1a3f6 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -44,7 +44,12 @@ import java.net.InetSocketAddress; import java.net.URLDecoder; -import java.util.*; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; +import java.util.List; +import java.util.Properties; +import java.util.Iterator; /** * @author sishu.yss From 1eeea8adbc7a3a338bcce65fd78f001b3d0b382b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Apr 2020 11:27:10 +0800 Subject: [PATCH 094/378] =?UTF-8?q?add=20isHighAvailability=20=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/launcher/ClusterClientFactory.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 391e1a3f6..a40b0155c 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -60,6 +60,12 @@ public class ClusterClientFactory { private static final String HA_CLUSTER_ID = "high-availability.cluster-id"; + private static final String HIGH_AVAILABILITY = "high-availability"; + + private static final String NODE = "NONE"; + + private static final String ZOOKEEPER = "zookeeper"; + private static final String HADOOP_CONF = "fs.hdfs.hadoopconf"; public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { @@ -96,6 +102,8 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { + boolean isHighAvailability; + config.setString(HADOOP_CONF, yarnConfDir); FileSystem.initialize(config); @@ -123,7 +131,9 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { throw new RuntimeException("No flink session found on yarn cluster."); } - if (config.getString(HA_CLUSTER_ID, null) == null) { + isHighAvailability = config.getString(HIGH_AVAILABILITY, NODE).equals(ZOOKEEPER); + + if (isHighAvailability && config.getString(HA_CLUSTER_ID, null) == null) { config.setString(HA_CLUSTER_ID, applicationId.toString()); } From 90a77555d41a577b31e792bc0777d88df3703c3e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:34:55 +0800 Subject: [PATCH 095/378] fix --- .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 5 +---- .../dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index da2e881a6..650a22629 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -19,8 +19,6 @@ package com.dtstack.flink.sql.launcher; -//import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; - import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; @@ -50,7 +48,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -129,7 +126,7 @@ private static String[] parseJson(String[] args) { StringBuilder lastStr = new StringBuilder(); try { FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, Charsets.UTF_8); reader = new BufferedReader(inputStreamReader); String tempString; while ((tempString = reader.readLine()) != null) { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 287844c87..956495ef2 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -75,7 +75,6 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); - System.out.println(tips); LOG.info(tips); return applicationId; From 49159c7cf175d3a29d03519d9647a2610b069f4c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:42:18 +0800 Subject: [PATCH 096/378] fix --- .../perjob/PerJobClusterClientBuilder.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index c40f9432d..9556956eb 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public class PerJobClusterClientBuilder { public void init(String yarnConfDir, Configuration flinkConfig, Properties userConf) throws Exception { - if(Strings.isNullOrEmpty(yarnConfDir)) { + if (Strings.isNullOrEmpty(yarnConfDir)) { throw new RuntimeException("parameters of yarn is required"); } userConf.forEach((key, val) -> flinkConfig.setString(key.toString(), val.toString())); @@ -79,8 +79,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - System.out.println("----init yarn success ----"); -// LOG.info("----init yarn success ----"); + LOG.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) @@ -131,8 +130,8 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ private static void fillJobGraphClassPath(JobGraph jobGraph) throws MalformedURLException { Map jobCacheFileConfig = jobGraph.getUserArtifacts(); - for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ - if(tmp.getKey().startsWith("class_path")){ + for (Map.Entry tmp : jobCacheFileConfig.entrySet()) { + if (tmp.getKey().startsWith("class_path")) { jobGraph.getClasspaths().add(new URL("file:" + tmp.getValue().filePath)); } } @@ -141,8 +140,8 @@ private static void fillJobGraphClassPath(JobGraph jobGraph) throws MalformedURL private List getPluginPathToShipFiles(JobGraph jobGraph) { List shipFiles = new ArrayList<>(); Map jobCacheFileConfig = jobGraph.getUserArtifacts(); - for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ - if(tmp.getKey().startsWith("class_path")){ + for (Map.Entry tmp : jobCacheFileConfig.entrySet()) { + if (tmp.getKey().startsWith("class_path")) { shipFiles.add(new File(tmp.getValue().filePath)); } } From d7baa87ed2b6ae691aed67805134d9d8d545c684 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:46:44 +0800 Subject: [PATCH 097/378] fix --- .../com/dtstack/flink/sql/launcher/LauncherMain.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 3811d349d..750356db2 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -40,11 +40,12 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -60,6 +61,8 @@ public class LauncherMain { private static final String CORE_JAR = "core"; + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); + private static String SP = File.separator; private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { @@ -80,7 +83,9 @@ public static void main(String[] args) throws Exception { confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - if(mode.equals(ClusterMode.local.name())) { + LOG.info("current job mode is {}", mode); + + if (mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[0]); Main.main(localArgs); return; From c4ce9357300195d20e2a2a72fe59a99c3a4998c4 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 10 Apr 2020 10:43:35 +0800 Subject: [PATCH 098/378] add zk session expired error --- .../com/dtstack/flink/sql/enums/EConnectionErrorCode.java | 2 +- .../com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java index feed69785..a0a281c4a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EConnectionErrorCode.java @@ -30,7 +30,7 @@ */ public enum EConnectionErrorCode { ERROR_NOT_MATCH(0, "错误信息未匹配", new String[]{}), - CONN_DB_INVALID(1, "数据库连接失效，请重新打开", new String[]{"the last packet successfully received from the server was"}), + CONN_DB_INVALID(1, "数据库连接失效，请重新打开", new String[]{"the last packet successfully received from the server was", "Zookeeper session has been expired"}), CONN_DB_FAILED(2, "数据库连接失败，请检查用户名或密码是否正确", new String[]{"Access denied for user"}), DB_TABLE_NOT_EXIST(3, "操作的表名不存在", new String[]{"doesn't exist"}); diff --git a/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java index a43b253df..6ae55c425 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/EConnectionErrorCodeTest.java @@ -34,6 +34,10 @@ public void testResolveErrorCodeFromException(){ EConnectionErrorCode errorCode = EConnectionErrorCode.resolveErrorCodeFromException(new Exception("The last packet successfully received from the server was 179 milliseconds")); + EConnectionErrorCode ckSessionExpired = + EConnectionErrorCode.resolveErrorCodeFromException(new Exception("Excepetion: Zookeeper session has been expired")); + Assert.assertEquals(errorCode, EConnectionErrorCode.CONN_DB_INVALID); + Assert.assertEquals(ckSessionExpired, EConnectionErrorCode.CONN_DB_INVALID); } } From 4a4f21c0d31fe6614ddd94c741abfd88daa5ff30 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 10 Apr 2020 12:00:19 +0800 Subject: [PATCH 099/378] rdb async connection pool size --- .../clickhouse/ClickhouseAsyncReqRow.java | 4 +-- .../dtstack/flink/sql/side/SideTableInfo.java | 30 +++++++++++++++++++ .../flink/sql/table/AbsSideTableParser.java | 8 +++++ .../flink/sql/side/db2/Db2AsyncReqRow.java | 4 +-- .../sql/side/impala/ImpalaAsyncReqRow.java | 29 ++++++++---------- .../sql/side/mysql/MysqlAsyncReqRow.java | 4 +-- .../sql/side/oracle/OracleAsyncReqRow.java | 4 +-- .../sql/side/polardb/PolardbAsyncReqRow.java | 4 +-- .../postgresql/PostgresqlAsyncReqRow.java | 4 +-- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 8 +++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 13 +++++++- .../sql/side/rdb/table/RdbSideTableInfo.java | 12 ++++++++ .../side/sqlserver/SqlserverAsyncReqRow.java | 4 +-- 13 files changed, 96 insertions(+), 32 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 2b5223412..db49e4337 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -48,7 +48,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { System.setProperty("vertx.disableFileCPResolving", "true"); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index 8c78578fb..32f065123 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -55,6 +55,8 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; + private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; @@ -65,6 +67,11 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { private int asyncTimeout=10000; + /** + * async operator req outside conn pool size, egg rdb conn pool size + */ + private int asyncPoolSize = 0; + private int asyncTimeoutNumLimit = Integer.MAX_VALUE; private boolean partitionedJoin = false; @@ -156,4 +163,27 @@ public void setAsyncTimeoutNumLimit(int asyncTimeoutNumLimit) { this.asyncTimeoutNumLimit = asyncTimeoutNumLimit; } + public int getAsyncPoolSize() { + return asyncPoolSize; + } + + public void setAsyncPoolSize(int asyncPoolSize) { + this.asyncPoolSize = asyncPoolSize; + } + + @Override + public String toString() { + return "Cache Info{" + + "cacheType='" + cacheType + '\'' + + ", cacheSize=" + cacheSize + + ", cacheTimeout=" + cacheTimeout + + ", asyncCapacity=" + asyncCapacity + + ", asyncTimeout=" + asyncTimeout + + ", asyncPoolSize=" + asyncPoolSize + + ", asyncTimeoutNumLimit=" + asyncTimeoutNumLimit + + ", partitionedJoin=" + partitionedJoin + + ", cacheMode='" + cacheMode + '\'' + + '}'; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index bf0b8a7a4..403360400 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.MathUtil; +import org.apache.flink.util.Preconditions; + import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,6 +116,12 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } + + if (props.containsKey(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { + Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); + Preconditions.checkArgument(asyncPoolSize > 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); + sideTableInfo.setAsyncPoolSize(asyncPoolSize); + } } } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 3d85c6fbd..cda2ed128 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -59,7 +59,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); db2lientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", DB2_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 725e7c3ee..2b76ec26c 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -57,33 +57,28 @@ public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List>> newCache = Maps.newConcurrentMap(); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2d767ccd9..971285e52 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.dtstack.flink.sql.util.DateUtil; import io.vertx.core.json.JsonArray; @@ -59,7 +60,9 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static int DEFAULT_VERTX_WORKER_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; + + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; @@ -73,6 +76,14 @@ public class RdbAsyncReqRow extends AsyncReqRow { public RdbAsyncReqRow(SideInfo sideInfo) { super(sideInfo); + init(sideInfo); + } + + protected void init(SideInfo sideInfo) { + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); + int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; + rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index 2c8c18fcd..fdb024b1d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -101,4 +101,16 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } + + @Override + public String toString() { + String cacheInfo = super.toString(); + String connectionInfo = "Rdb Side Connection Info{" + + "url='" + url + '\'' + + ", tableName='" + tableName + '\'' + + ", schema='" + schema + '\'' + + '}'; + return cacheInfo + " , " + connectionInfo; + } + } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index cb4240ca6..9fc2c35cd 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -51,7 +51,7 @@ public void open(Configuration parameters) throws Exception { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", SQLSERVER_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) + .put("max_pool_size", rdbSideTableInfo.getAsyncPoolSize()) .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) @@ -64,7 +64,7 @@ public void open(Configuration parameters) throws Exception { VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); + vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); From 330be96a729c630506a8737ce198d5ee6a77b99e Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 10 Apr 2020 12:56:06 +0800 Subject: [PATCH 100/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=AE=B0=E8=BF=B0?= =?UTF-8?q?=E5=92=8C=E8=B6=85=E8=BF=87=E6=AC=A1=E6=95=B0=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/table/AbstractSideTableParser.java | 6 ---- .../sql/side/rdb/async/RdbAsyncReqRow.java | 33 ++++++++++--------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 7df86b3cb..de46e8409 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -108,12 +108,6 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ - sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); - } - } if(props.containsKey(AbstractSideTableInfo.ASYNC_FAIL_MAX_NUM_KEY.toLowerCase())){ Long asyncFailNum = MathUtil.getLongVal(props.get(AbstractSideTableInfo.ASYNC_FAIL_MAX_NUM_KEY.toLowerCase())); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index c86434528..ca6dcc38b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -31,6 +32,7 @@ import io.vertx.ext.sql.SQLConnection; import org.apache.calcite.sql.JoinType; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -77,10 +79,12 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); - private final static AtomicLong TIMOUT_NUM = new AtomicLong(0); + private final static AtomicLong FAIL_NUM = new AtomicLong(0); private Logger logger = LoggerFactory.getLogger(getClass()); + private Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_SIDE_PARSE_ERROR_RECORDS); + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @@ -123,7 +127,12 @@ private void connectWithRetry(Map inputParams, CRow input, Resul logger.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ - outByJoinType(resultFuture, conn.cause()); + if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + counter.inc(); + resultFuture.completeExceptionally(conn.cause()); + } else { + dealMissKey(input, resultFuture); + } finishFlag.set(true); } conn.result().close(); @@ -135,6 +144,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul handleQuery(conn.result(), inputParams, input, resultFuture); finishFlag.set(true); } catch (Exception e) { + dealFillDataError(resultFuture, e, null); logger.error("", e); } finally { latch.countDown(); @@ -200,12 +210,13 @@ private void handleQuery(SQLConnection connection,Map inputParam JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { if (rs.failed()) { - if(TIMOUT_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ - outByJoinType(resultFuture, rs.cause()); - return; + if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + LOG.error("Cannot retrieve the data from the database", rs.cause()); + counter.inc(); + resultFuture.completeExceptionally(rs.cause()); + } else { + dealMissKey(input, resultFuture); } - LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.complete(null); return; } @@ -244,12 +255,4 @@ private void handleQuery(SQLConnection connection,Map inputParam }); } - private void outByJoinType(ResultFuture resultFuture, Throwable e){ - if(sideInfo.getJoinType() == JoinType.LEFT){ - resultFuture.complete(null); - return; - } - resultFuture.completeExceptionally(e); - } - } From 2027b415a930d35417575669a31212b5f5b93523 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 13 Apr 2020 19:08:49 +0800 Subject: [PATCH 101/378] parse where case --- .../flink/sql/parser/InsertSqlParser.java | 16 +++------- .../flink/sql/side/SidePredicatesParser.java | 21 +++++++------ .../flink/sql/util/FieldReplaceUtil.java | 17 ++++++++++- .../dtstack/flink/sql/util/TableUtils.java | 30 ++++++++++++------- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index e2940c4f7..e29ac4bef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -26,6 +26,7 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; +import org.apache.flink.table.calcite.FlinkPlannerImpl; import java.util.List; @@ -53,18 +54,9 @@ public static InsertSqlParser newInstance(){ @Override public void parseSql(String sql, SqlTree sqlTree) { - SqlParser.Config config = SqlParser - .configBuilder() - .setLex(Lex.MYSQL) - .build(); - - SqlParser sqlParser = SqlParser.create(sql,config); - SqlNode sqlNode = null; - try { - sqlNode = sqlParser.parseStmt(); - } catch (SqlParseException e) { - throw new RuntimeException("", e); - } + + FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); + SqlNode sqlNode = flinkPlanner.parse(sql); SqlParseResult sqlParseResult = new SqlParseResult(); parseNode(sqlNode, sqlParseResult); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 0902bf39f..99dc3507b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -138,16 +138,19 @@ private void extractPredicateInfo(SqlNode whereNode, List predica private void fillPredicateInfoToList(SqlBasicCall whereNode, List predicatesInfoList, String operatorName, SqlKind operatorKind, int fieldIndex, int conditionIndex) { - SqlIdentifier fieldFullPath = (SqlIdentifier) whereNode.getOperands()[fieldIndex]; - if (fieldFullPath.names.size() == 2) { - String ownerTable = fieldFullPath.names.get(0); - String fieldName = fieldFullPath.names.get(1); - String content = (operatorKind == SqlKind.BETWEEN) ? whereNode.getOperands()[conditionIndex].toString() + " AND " + - whereNode.getOperands()[2].toString() : whereNode.getOperands()[conditionIndex].toString(); + SqlNode sqlNode = whereNode.getOperands()[fieldIndex]; + if (sqlNode.getKind() == SqlKind.IDENTIFIER) { + SqlIdentifier fieldFullPath = (SqlIdentifier) sqlNode; + if (fieldFullPath.names.size() == 2) { + String ownerTable = fieldFullPath.names.get(0); + String fieldName = fieldFullPath.names.get(1); + String content = (operatorKind == SqlKind.BETWEEN) ? whereNode.getOperands()[conditionIndex].toString() + " AND " + + whereNode.getOperands()[2].toString() : whereNode.getOperands()[conditionIndex].toString(); - PredicateInfo predicateInfo = PredicateInfo.builder().setOperatorName(operatorName).setOperatorKind(operatorKind.toString()) - .setOwnerTable(ownerTable).setFieldName(fieldName).setCondition(content).build(); - predicatesInfoList.add(predicateInfo); + PredicateInfo predicateInfo = PredicateInfo.builder().setOperatorName(operatorName).setOperatorKind(operatorKind.toString()) + .setOwnerTable(ownerTable).setFieldName(fieldName).setCondition(content).build(); + predicatesInfoList.add(predicateInfo); + } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index 10919ca5b..697954dd6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -146,7 +146,22 @@ private static SqlNode replaceNodeInfo(SqlNode groupNode, } return sqlBasicCall; - }else{ + } else if (groupNode.getKind() == CASE) { + SqlCase sqlCase = (SqlCase) groupNode; + + for (int i = 0; i < sqlCase.getWhenOperands().size(); i++) { + SqlNode sqlNode = sqlCase.getWhenOperands().getList().get(i); + SqlNode replaceNode = replaceSelectFieldName(sqlNode, oldTbName, newTbName, mappingField); + sqlCase.getWhenOperands().set(i,replaceNode); + } + + for (int i = 0; i < sqlCase.getThenOperands().size(); i++) { + SqlNode sqlNode = sqlCase.getThenOperands().getList().get(i); + SqlNode replaceNode = replaceSelectFieldName(sqlNode, oldTbName, newTbName, mappingField); + sqlCase.getThenOperands().set(i,replaceNode); + } + return sqlCase; + } else { return groupNode; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 1527c2db7..579beaf63 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -568,7 +568,12 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S }else if(selectNode.getKind() == OTHER){ //不处理 return; - }else{ + } else if (selectNode.getKind() == CASE) { + SqlCase sqlCase = (SqlCase) selectNode; + + sqlCase.getWhenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); + sqlCase.getThenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); + } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } } @@ -579,14 +584,14 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S * @param fieldInfos */ public static void getConditionRefTable(SqlNode selectNode, Set fieldInfos) { - if(selectNode.getKind() == IDENTIFIER){ + if (selectNode.getKind() == IDENTIFIER) { SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; fieldInfos.add(sqlIdentifier.toString()); return; - }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 + } else if (selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN) {//字面含义 return; - }else if( AGGREGATE.contains(selectNode.getKind()) + } else if (AGGREGATE.contains(selectNode.getKind()) || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) || COMPARISON.contains(selectNode.getKind()) || selectNode.getKind() == OTHER_FUNCTION @@ -616,15 +621,15 @@ public static void getConditionRefTable(SqlNode selectNode, Set fieldInf || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE - ){ + ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; - for(int i=0; i fieldInf } return; - }else if(selectNode.getKind() == OTHER){ + } else if (selectNode.getKind() == OTHER) { //不处理 return; - }else{ + } else if (selectNode.getKind() == CASE) { + SqlCase sqlCase = (SqlCase) selectNode; + + sqlCase.getWhenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); + sqlCase.getThenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); + } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } } From a7ea7f3c8eda717d50c9e1edf8baefe883686363 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 13 Apr 2020 21:34:38 +0800 Subject: [PATCH 102/378] add log --- core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index fae0ba9aa..6f0dce58a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -66,6 +66,7 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); initMetric(); + LOG.info("async dim table config info: {} ", sideInfo.getSideTableInfo().toString()); } private void initCache(){ From 63f3ab158c2900deed4063c123a08f12754f1430 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 14 Apr 2020 10:05:10 +0800 Subject: [PATCH 103/378] fix spelling and kafka params map name --- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 10 +++++----- .../flink/sql/sink/kafka/table/KafkaSinkTableInfo.java | 8 ++++---- .../sql/source/kafka/table/KafkaSourceTableInfo.java | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 7ffe7d1dc..4bd33363c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -266,7 +266,7 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { - Set pluginClassPatshSets = Sets.newHashSet(); + Set pluginClassPathSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { @@ -304,7 +304,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment registerTableCache.put(tableInfo.getName(), regTable); URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sourceTablePathUrl); + pluginClassPathSets.add(sourceTablePathUrl); } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); @@ -312,18 +312,18 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sinkTablePathUrl); + pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); - pluginClassPatshSets.add(sideTablePathUrl); + pluginClassPathSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); } } - return pluginClassPatshSets; + return pluginClassPathSets; } /** diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java index f1bef088e..2804c71de 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java @@ -58,7 +58,7 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String bootstrapServers; - public Map kafkaParam = new HashMap(); + public Map kafkaParams = new HashMap(); private String topic; @@ -73,15 +73,15 @@ public class KafkaSinkTableInfo extends AbstractTargetTableInfo { private String updateMode; public void addKafkaParam(String key, String value) { - kafkaParam.put(key, value); + kafkaParams.put(key, value); } public String getKafkaParam(String key) { - return kafkaParam.get(key); + return kafkaParams.get(key); } public Set getKafkaParamKeys() { - return kafkaParam.keySet(); + return kafkaParams.keySet(); } public String getBootstrapServers() { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 1210aba40..62453166f 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -69,7 +69,7 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { private String fieldDelimiter; - public Map kafkaParam = new HashMap<>(); + public Map kafkaParams = new HashMap<>(); public String getBootstrapServers() { @@ -113,15 +113,15 @@ public void setTopicIsPattern(Boolean topicIsPattern) { } public void addKafkaParam(Map kafkaParam) { - kafkaParam.putAll(kafkaParam); + kafkaParams.putAll(kafkaParam); } public String getKafkaParam(String key) { - return kafkaParam.get(key); + return kafkaParams.get(key); } public Set getKafkaParamKeys() { - return kafkaParam.keySet(); + return kafkaParams.keySet(); } public String getSourceDataType() { From 86bd4b6a06a925f05955fcdb6e61263063e6316d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 14 Apr 2020 11:41:04 +0800 Subject: [PATCH 104/378] fix 24602 --- .../flink/sql/side/rdb/all/RdbAllReqRow.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 60e3d817f..2fc1d3b23 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -83,12 +83,12 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); - cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { throw new RuntimeException(e); } + cacheRef.set(newCache); LOG.info("----- rdb all cacheRef reload end:{}", Calendar.getInstance()); } @@ -112,9 +112,9 @@ public void flatMap(CRow value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(cacheKey); if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { out.collect(new CRow(fillData(value.row(), null), value.change())); + } else if (!CollectionUtils.isEmpty(cacheList)) { + cacheList.forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } - - cacheList.stream().forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); } @Override @@ -141,8 +141,8 @@ public Row fillData(Row input, Object sideInput) { } /** - * covert flink time attribute.Type information for indicating event or processing time. - * However, it behaves like a regular SQL timestamp but is serialized as Long. + * covert flink time attribute.Type information for indicating event or processing time. + * However, it behaves like a regular SQL timestamp but is serialized as Long. * * @param entry * @param obj @@ -222,7 +222,8 @@ public int getFetchSize() { } /** - * get jdbc connection + * get jdbc connection + * * @param dbURL * @param userName * @param password From 6943b54f1b9964cbceeae35a8b2309c1fb08b249 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 14 Apr 2020 15:54:21 +0800 Subject: [PATCH 105/378] =?UTF-8?q?fix=20=E5=88=AB=E5=90=8D=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideInfo.java | 7 ++++--- .../dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index 029c86e25..d8a22e022 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -93,7 +93,7 @@ public void parseSelectFields(JoinInfo joinInfo){ if(fieldInfo.getTable().equalsIgnoreCase(sideTableName)){ String sideFieldName = sideTableInfo.getPhysicalFields().getOrDefault(fieldInfo.getFieldName(), fieldInfo.getFieldName()); fields.add(sideFieldName); - sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(sideFieldName)); + sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(fieldInfo.getFieldName())); sideFieldIndex.put(i, sideTableFieldIndex); sideFieldNameIndex.put(i, sideFieldName); sideTableFieldIndex++; @@ -132,12 +132,13 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); + Map physicalFields = sideTableInfo.getPhysicalFields(); String rightTableName = right.getComponent(0).getSimple(); String rightField = right.getComponent(1).getSimple(); if(leftTableName.equalsIgnoreCase(sideTableName)){ - equalFieldList.add(leftField); + equalFieldList.add(physicalFields.get(leftField)); int equalFieldIndex = -1; for(int i=0; i physicalFields = sideTableInfo.getPhysicalFields(); String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); @@ -94,7 +96,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { String rightField = right.getComponent(1).getSimple(); if (leftTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(leftField); + equalFieldList.add(physicalFields.get(leftField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -110,7 +112,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } else if (rightTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(rightField); + equalFieldList.add(physicalFields.get(rightField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; From 75adb0b0cbb56e57797d915a9aadfe87ee0dc635 Mon Sep 17 00:00:00 2001 From: maqi Date: Tue, 14 Apr 2020 17:04:14 +0800 Subject: [PATCH 106/378] side predicate skip where case --- .../com/dtstack/flink/sql/side/SidePredicatesParser.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 99dc3507b..4d6112aa8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -30,6 +30,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.table.calcite.FlinkPlannerImpl; import java.util.List; @@ -147,6 +148,10 @@ private void fillPredicateInfoToList(SqlBasicCall whereNode, List String content = (operatorKind == SqlKind.BETWEEN) ? whereNode.getOperands()[conditionIndex].toString() + " AND " + whereNode.getOperands()[2].toString() : whereNode.getOperands()[conditionIndex].toString(); + if (StringUtils.containsIgnoreCase(content,SqlKind.CASE.toString())) { + return; + } + PredicateInfo predicateInfo = PredicateInfo.builder().setOperatorName(operatorName).setOperatorKind(operatorKind.toString()) .setOwnerTable(ownerTable).setFieldName(fieldName).setCondition(content).build(); predicatesInfoList.add(predicateInfo); From 9bf9a659b25dc4b3c22511e53cef7fe00e3def6b Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 14 Apr 2020 17:46:52 +0800 Subject: [PATCH 107/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcoalesce=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E4=B8=8D=E8=83=BD=E7=94=A8=E9=97=AE=E9=A2=98=E3=80=90?= =?UTF-8?q?http://redmine.prod.dtstack.cn/issues/24635=3Fissue=5Fcount=3D2?= =?UTF-8?q?&issue=5Fposition=3D1&next=5Fissue=5Fid=3D24600=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index 10919ca5b..4b05840d6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -253,6 +253,7 @@ public static SqlNode replaceSelectFieldName(SqlNode selectNode, || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; From 04550b05a4b7c47d56536f6b75d86ac17f0b2bbf Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 15 Apr 2020 14:44:58 +0800 Subject: [PATCH 108/378] =?UTF-8?q?=E5=A4=84=E7=90=86=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/AbstractSideTableInfo.java | 2 -- .../flink/sql/side/BaseAsyncReqRow.java | 23 +++++++------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 30 +++++-------------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 71a309411..19d3cc5fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -54,8 +54,6 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_KEY = "asyncTimeout"; - public static final String ASYNC_TIMEOUT_NUM_KEY = "asyncTimeoutNum"; - public static final String ASYNC_FAIL_MAX_NUM_KEY = "asyncFailMaxNum"; private String cacheType = "none";//None or LRU or ALL diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index bfe14c3ea..7f8478183 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -51,6 +51,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -64,10 +65,9 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private RuntimeContext runtimeContext; - + private final static AtomicLong FAIL_NUM = new AtomicLong(0); private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; - protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; @@ -125,7 +125,7 @@ protected void dealMissKey(CRow input, ResultFuture resultFuture){ Row row = fillData(input.row(), null); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } }else{ resultFuture.complete(null); @@ -205,7 +205,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result Row row = fillData(input.row(), val); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else if (ECacheContentType.MultiLine == val.getType()) { try { @@ -216,7 +216,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result } resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else { resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); @@ -256,11 +256,14 @@ protected void cancelTimerWhenComplete(ResultFuture resultFuture, Schedule } } - protected void dealFillDataError(ResultFuture resultFuture, Exception e, Object sourceData) { - LOG.debug("source data {} join side table error ", sourceData); - LOG.debug("async buid row error..{}", e); - parseErrorRecords.inc(); - resultFuture.complete(Collections.emptyList()); + protected void dealFillDataError(CRow input, ResultFuture resultFuture, Throwable e) { + if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + LOG.info("dealFillDataError", e); + parseErrorRecords.inc(); + resultFuture.completeExceptionally(e); + } else { + dealMissKey(input, resultFuture); + } } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index ca6dcc38b..9571d7962 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -30,9 +29,7 @@ import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; -import org.apache.calcite.sql.JoinType; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -79,12 +76,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); - private final static AtomicLong FAIL_NUM = new AtomicLong(0); - private Logger logger = LoggerFactory.getLogger(getClass()); - private Counter counter = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_SIDE_PARSE_ERROR_RECORDS); - public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @@ -98,7 +91,12 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { + AtomicLong networkLogCounter = new AtomicLong(0L); while (!CONN_STATUS.get()){//network is unhealth + //todo:统一计数 + if(networkLogCounter.getAndIncrement() % 1000 == 0){ + LOG.info("network unhealth to block task"); + } Thread.sleep(100); } rdbSqlClient.getConnection(conn -> { @@ -127,12 +125,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul logger.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ - if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ - counter.inc(); - resultFuture.completeExceptionally(conn.cause()); - } else { - dealMissKey(input, resultFuture); - } + dealFillDataError(input, resultFuture, conn.cause()); finishFlag.set(true); } conn.result().close(); @@ -144,8 +137,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul handleQuery(conn.result(), inputParams, input, resultFuture); finishFlag.set(true); } catch (Exception e) { - dealFillDataError(resultFuture, e, null); - logger.error("", e); + dealFillDataError(input, resultFuture, e); } finally { latch.countDown(); } @@ -210,13 +202,7 @@ private void handleQuery(SQLConnection connection,Map inputParam JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { if (rs.failed()) { - if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ - LOG.error("Cannot retrieve the data from the database", rs.cause()); - counter.inc(); - resultFuture.completeExceptionally(rs.cause()); - } else { - dealMissKey(input, resultFuture); - } + dealFillDataError(input, resultFuture, rs.cause()); return; } From 96936193e3e18ad7d85b536401afffd30d95c8c1 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 15 Apr 2020 15:29:49 +0800 Subject: [PATCH 109/378] update doc quickStart --- README.md | 4 +- docs/quickStart.md | 224 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 210 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e5d4622eb..30c95552c 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ FlinkStreamSQL * 自定义create view 语法 * 自定义create function 语法 * 实现了流与维表的join - * 支持原生FLinkSQL所有的语法 - * 扩展了输入和输出的性能指标到promethus + * 支持原生FlinkSQL所有的语法 + * 扩展了输入和输出的性能指标到Task metrics ## 目录 diff --git a/docs/quickStart.md b/docs/quickStart.md index dda046062..f1607c5e7 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -8,7 +8,7 @@ ### 1.2 执行环境 * Java: JDK8及以上 -* Flink集群: 1.4,1.5,1.8（单机模式不需要安装Flink集群） +* Flink集群: 1.4,1.5,1.8,1.9,1.10（单机模式不需要安装Flink集群） * 操作系统：理论上不限 * kerberos环境需要在flink-conf.yaml配置security.kerberos.login.keytab以及security.kerberos.login.principal参数，配置案例: ``` @@ -29,7 +29,6 @@ zookeeper.sasl.login-context-name: Client ``` mvn clean package -Dmaven.test.skip - ``` ####可运行的目录结构: @@ -37,9 +36,9 @@ mvn clean package -Dmaven.test.skip | |-----bin | |--- submit.sh 任务启动脚本 -|-----lib -| |--- sql.launcher.jar 包存储路径，是任务提交的入口（需要手动移动到该目录） -|-----plugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) +|-----lib: launcher包存储路径，是任务提交的入口 +| |--- sql.launcher.jar +|-----sqlplugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) | |--- core.jar | |--- xxxsource | |--- xxxsink @@ -49,15 +48,208 @@ mvn clean package -Dmaven.test.skip #### 1.4.1 启动命令 +```shell script +# 脚本启动 +sh submit.sh + -mode yarn + -name flink1.10_yarnSession + -sql F:\dtstack\stressTest\flinkStreamSql\1.10_dev\sql\flink1100.sql + -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + -remoteSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + -flinkconf F:\dtstack\flink\flink-1.10.0\conf + -yarnconf F:\dtstack\flinkStreamSql\yarnConf_node1 + -flinkJarPath F:\dtstack\flink\flink-1.10.0\lib + -pluginLoadMode shipfile + -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} + -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} +``` +```shell script +# 通过idea启动 程序入口类LaucherMain +# Run/Debug Configurations中设置Program arguments +-mode yarnPer +-sql /home/wen/Desktop/flink_stream_sql_conf/sql/stressTest.sql +-name stressTestAll +-localSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins +-remoteSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins +-flinkconf /home/wen/Desktop/flink_stream_sql_conf/flinkConf +-yarnconf /home/wen/Desktop/flink_stream_sql_conf/yarnConf_node1 +-flinkJarPath /home/wen/Desktop/dtstack/flink-1.8.1/lib +-pluginLoadMode shipfile +-confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} +-queue c +``` +#### 1.4.2 命令参数说明 +* **mode** + * 描述：执行模式，也就是flink集群的工作模式 + * local: 本地模式 + * standalone: 提交到独立部署模式的flink集群 + * yarn: 提交到yarn模式的flink集群，该模式下需要提前启动一个yarn-session，使用默认名"Flink session cluster" + * yarnPer: yarn per_job模式提交(即创建新flink application)，默认名为flink任务名称 + * 必选：否 + * 默认值：local + +* **name** + * 描述：flink 任务对应名称。 + * 必选：是 + * 默认值：无 + +* **sql** + * 描述：待执行的flink sql所在路径 。 + * 必选：是 + * 默认值：无 + +* **localSqlPluginPath** + * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 + * 必选：是 + * 默认值：无 + +* **remoteSqlPluginPath** + * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 + * 必选：否 + * 默认值：无 + +* **addjar** + * 描述：扩展jar路径,当前主要是UDF定义的jar； + * 必选：否 + * 默认值：无 + +* **confProp** + * 描述：一些参数设置 + * 必选：是 （如无参数填写空json即可） + * 默认值：{} + * 可选参数: + * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) + * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 + * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 + * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 + * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 + * sql.env.parallelism: 默认并行度设置 + * sql.max.env.parallelism: 最大并行度设置 + * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] + * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) + * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] + * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) + * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 + * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] + * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// + * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) + * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) + * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) + * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) + * savePointPath：任务恢复点的路径（默认无） + * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） + * restore.enable：是否失败重启（默认是true） + * failure.interval：衡量失败率的时间段，单位分钟（默认6m） + * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） + * logLevel: 日志级别动态配置（默认info） + * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 + + +* **flinkconf** + * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.10.0/conf + * 必选：否 + * 默认值：无 + +* **yarnconf** + * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop + * 必选：否 + * 默认值：无 + +* **flinkJarPath** + * 描述：per_job 模式提交需要指定本地的flink jar存放路径 + * 必选：否 + * 默认值：无 + +* **queue** + * 描述：per_job 模式下指定的yarn queue + * 必选：否 + * 默认值：default + +* **pluginLoadMode** + * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 + * 必选：否 + * 默认值：classpath + +* **yarnSessionConf** + * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid + * 必选：否 + * 默认值：{} + +## 1.5 任务样例 + ``` -sh submit.sh -sql D:\sideSql.txt --name xctest --remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin --localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins --addjar \["udf.jar\"\] --mode yarn --flinkconf D:\flink_home\kudu150etc --yarnconf D:\hadoop\etc\hadoopkudu --confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} --yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} -``` \ No newline at end of file +# 一个kafka join all维表 sink kafka的样例 +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( + id bigint, + school varchar, + home varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/tiezhu', + userName='dtstack', + password='you-guess', + tableName='stressTest', + cache='ALL', + parallelism='1', + asyncCapacity='100' +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + home varchar, + school varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_out', + parallelism = '1' +); + + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.school AS school, + t2.home AS home + from + ( + select + id, + name, + address + from + MyTable + ) t1 + left join sideTable t2 + on t2.id = t2.id; +``` + +# 招聘 +1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 + From e496e3376be02a05badcc88cea7d0d2950fa9eda Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 15 Apr 2020 15:37:24 +0800 Subject: [PATCH 110/378] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=AA=E5=88=AB?= =?UTF-8?q?=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index f1607c5e7..52fc3af4e 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -22,11 +22,8 @@ zookeeper.sasl.service-name: zookeeper zookeeper.sasl.login-context-name: Client ``` - ### 1.3 打包 - 进入项目根目录，使用maven打包： - ``` mvn clean package -Dmaven.test.skip ``` @@ -38,13 +35,13 @@ mvn clean package -Dmaven.test.skip | |--- submit.sh 任务启动脚本 |-----lib: launcher包存储路径，是任务提交的入口 | |--- sql.launcher.jar -|-----sqlplugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) +|-----plugins: 插件包存储路径(mvn 打包之后会自动将jar移动到该目录下) | |--- core.jar | |--- xxxsource | |--- xxxsink | |--- xxxside ``` -### 1.4 启动 +### 1.4 快速启动 #### 1.4.1 启动命令 @@ -115,7 +112,7 @@ sh submit.sh * **confProp** * 描述：一些参数设置 - * 必选：是 （如无参数填写空json即可） + * 必选：否 * 默认值：{} * 可选参数: * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) @@ -176,9 +173,8 @@ sh submit.sh * 默认值：{} ## 1.5 任务样例 - ``` -# 一个kafka join all维表 sink kafka的样例 +# 一个 kafka source join all维表 到 kafka sink的样例 CREATE TABLE MyTable( id bigint, name varchar, From 88d73ed3122c3f5cede49eef0abfb8cf2864e36e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 09:42:52 +0800 Subject: [PATCH 111/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9config=E4=B8=AD?= =?UTF-8?q?=E9=83=A8=E5=88=86=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/config.md b/docs/config.md index 48aeefc5a..3fb6ac723 100644 --- a/docs/config.md +++ b/docs/config.md @@ -9,8 +9,8 @@ sh submit.sh -key1 val1 -key2 val2 * 描述：执行模式，也就是flink集群的工作模式 * local: 本地模式 * standalone: 提交到独立部署模式的flink集群 - * yarn: 提交到yarn模式的flink集群(即提交到已有flink集群) - * yarnPer: yarn per_job模式提交(即创建新flink application) + * yarn: 提交到yarn模式的flink集群，该模式下需要提前启动一个yarn-session，使用默认名"Flink session cluster" + * yarnPer: yarn per_job模式提交(即创建新flink application)，默认名为flink任务名称 * 必选：否 * 默认值：local @@ -28,7 +28,7 @@ sh submit.sh -key1 val1 -key2 val2 * 默认值：无 * **sql** - * 描述：执行flink sql 的主体语句。 + * 描述：待执行的flink sql所在路径 。 * 必选：是 * 默认值：无 @@ -44,15 +44,13 @@ sh submit.sh -key1 val1 -key2 val2 * **addjar** * 描述：扩展jar路径,当前主要是UDF定义的jar； - * 格式：json * 必选：否 * 默认值：无 * **confProp** * 描述：一些参数设置 - * 格式: json - * 必选：是 （如无参数填写空json即可） - * 默认值：无 + * 必选：否 + * 默认值：{} * 可选参数: * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 @@ -79,7 +77,7 @@ sh submit.sh -key1 val1 -key2 val2 * **flinkconf** - * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.4.0/conf + * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.10.0/conf * 必选：否 * 默认值：无 @@ -91,15 +89,15 @@ sh submit.sh -key1 val1 -key2 val2 * **flinkJarPath** * 描述：yarnPer 模式提交需要指定本地的flink jar存放路径 * 必选：否 - * 默认值：false + * 默认值：无 * **queue** * 描述：yarnPer 模式下指定的yarn queue * 必选：否 - * 默认值：false + * 默认值：default * **yarnSessionConf** * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid * 必选：否 - * 默认值：false + * 默认值：{} \ No newline at end of file From 9b451a8ef1ef906e9efef9f5d95050cb47719713 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 09:59:32 +0800 Subject: [PATCH 112/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9quickStart=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 202 ++++----------------------------------------- 1 file changed, 16 insertions(+), 186 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index 52fc3af4e..a51f234e3 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -11,7 +11,7 @@ * Flink集群: 1.4,1.5,1.8,1.9,1.10（单机模式不需要安装Flink集群） * 操作系统：理论上不限 * kerberos环境需要在flink-conf.yaml配置security.kerberos.login.keytab以及security.kerberos.login.principal参数，配置案例: -``` +```yaml #提交到hadoop环境一定要配置fs.hdfs.hadoopconf参数 fs.hdfs.hadoopconf: /Users/maqi/tmp/hadoopconf/hadoop_250 security.kerberos.login.use-ticket-cache: true @@ -24,7 +24,7 @@ zookeeper.sasl.login-context-name: Client ### 1.3 打包 进入项目根目录，使用maven打包： -``` +```shell script mvn clean package -Dmaven.test.skip ``` @@ -59,193 +59,23 @@ sh submit.sh -pluginLoadMode shipfile -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} -``` -```shell script -# 通过idea启动 程序入口类LaucherMain -# Run/Debug Configurations中设置Program arguments --mode yarnPer --sql /home/wen/Desktop/flink_stream_sql_conf/sql/stressTest.sql --name stressTestAll --localSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins --remoteSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins --flinkconf /home/wen/Desktop/flink_stream_sql_conf/flinkConf --yarnconf /home/wen/Desktop/flink_stream_sql_conf/yarnConf_node1 --flinkJarPath /home/wen/Desktop/dtstack/flink-1.8.1/lib --pluginLoadMode shipfile --confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} --queue c -``` -#### 1.4.2 命令参数说明 -* **mode** - * 描述：执行模式，也就是flink集群的工作模式 - * local: 本地模式 - * standalone: 提交到独立部署模式的flink集群 - * yarn: 提交到yarn模式的flink集群，该模式下需要提前启动一个yarn-session，使用默认名"Flink session cluster" - * yarnPer: yarn per_job模式提交(即创建新flink application)，默认名为flink任务名称 - * 必选：否 - * 默认值：local - -* **name** - * 描述：flink 任务对应名称。 - * 必选：是 - * 默认值：无 -* **sql** - * 描述：待执行的flink sql所在路径 。 - * 必选：是 - * 默认值：无 - -* **localSqlPluginPath** - * 描述：本地插件根目录地址，也就是打包后产生的plugins目录。 - * 必选：是 - * 默认值：无 - -* **remoteSqlPluginPath** - * 描述：flink执行集群上的插件根目录地址（将打包好的插件存放到各个flink节点上,如果是yarn集群需要存放到所有的nodemanager上）。 - * 必选：否 - * 默认值：无 - -* **addjar** - * 描述：扩展jar路径,当前主要是UDF定义的jar； - * 必选：否 - * 默认值：无 - -* **confProp** - * 描述：一些参数设置 - * 必选：否 - * 默认值：{} - * 可选参数: - * sql.ttl.min: 最小过期时间,大于0的整数,如1d、1h(d\D:天,h\H:小时,m\M:分钟,s\s:秒) - * sql.ttl.max: 最大过期时间,大于0的整数,如2d、2h(d\D:天,h\H:小时,m\M:分钟,s\s:秒),需同时设置最小时间,且比最小时间大5分钟 - * state.backend: 任务状态后端，可选为MEMORY,FILESYSTEM,ROCKSDB，默认为flinkconf中的配置。 - * state.checkpoints.dir: FILESYSTEM,ROCKSDB状态后端文件系统存储路径，例如：hdfs://ns1/dtInsight/flink180/checkpoints。 - * state.backend.incremental: ROCKSDB状态后端是否开启增量checkpoint,默认为true。 - * sql.env.parallelism: 默认并行度设置 - * sql.max.env.parallelism: 最大并行度设置 - * time.characteristic: 可选值[ProcessingTime|IngestionTime|EventTime] - * sql.checkpoint.interval: 设置了该参数表明开启checkpoint(ms) - * sql.checkpoint.mode: 可选值[EXACTLY_ONCE|AT_LEAST_ONCE] - * sql.checkpoint.timeout: 生成checkpoint的超时时间(ms) - * sql.max.concurrent.checkpoints: 最大并发生成checkpoint数 - * sql.checkpoint.cleanup.mode: 默认是不会将checkpoint存储到外部存储,[true(任务cancel之后会删除外部存储)|false(外部存储需要手动删除)] - * flinkCheckpointDataURI: 设置checkpoint的外部存储路径,根据实际的需求设定文件路径,hdfs://, file:// - * jobmanager.memory.mb: per_job模式下指定jobmanager的内存大小(单位MB, 默认值:768) - * taskmanager.memory.mb: per_job模式下指定taskmanager的内存大小(单位MB, 默认值:768) - * taskmanager.num: per_job模式下指定taskmanager的实例数(默认1) - * taskmanager.slots：per_job模式下指定每个taskmanager对应的slot数量(默认1) - * savePointPath：任务恢复点的路径（默认无） - * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） - * restore.enable：是否失败重启（默认是true） - * failure.interval：衡量失败率的时间段，单位分钟（默认6m） - * delay.interval：连续两次重启尝试间的间隔，单位是秒（默认10s） - * logLevel: 日志级别动态配置（默认info） - * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 - - -* **flinkconf** - * 描述：flink配置文件所在的目录（单机模式下不需要），如/hadoop/flink-1.10.0/conf - * 必选：否 - * 默认值：无 - -* **yarnconf** - * 描述：Hadoop配置文件（包括hdfs和yarn）所在的目录（单机模式下不需要），如/hadoop/etc/hadoop - * 必选：否 - * 默认值：无 - -* **flinkJarPath** - * 描述：per_job 模式提交需要指定本地的flink jar存放路径 - * 必选：否 - * 默认值：无 - -* **queue** - * 描述：per_job 模式下指定的yarn queue - * 必选：否 - * 默认值：default - -* **pluginLoadMode** - * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 - * 必选：否 - * 默认值：classpath - -* **yarnSessionConf** - * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid - * 必选：否 - * 默认值：{} - -## 1.5 任务样例 +# mode: 任务启动的模式 +# name: 本次任务名称 +# sql: 本次任务执行sql脚本 +# localSqPluginPath: 本地插件包根目录地址 +# remoteSqlPluginPath: flink执行集群上的插件根目录地址 +# flinkconf: flink配置文件所在目录（单机模式下不需要） +# yarnconf: Hadoop配置文件（包括hdfs和yarn）所在目录 +# flinkJarPath: yarnPer模式提交需要指定本地的flink jar存放路径 +# pluginLoadMode:yarnPer模式下的插件包加载方式 +# confProp: 其他额外参数配置 +# yarnSessionConf: yarnSession模式下指定的运行参数，目前只支持指定yid ``` -# 一个 kafka source join all维表 到 kafka sink的样例 -CREATE TABLE MyTable( - id bigint, - name varchar, - address varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_in', - groupId = 'flink_sql', - timezone = 'Asia/Shanghai', - topicIsPattern = 'false', - parallelism = '1' -); - -CREATE TABLE sideTable( - id bigint, - school varchar, - home varchar, - PRIMARY KEY(id), - PERIOD FOR SYSTEM_TIME -)WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/tiezhu', - userName='dtstack', - password='you-guess', - tableName='stressTest', - cache='ALL', - parallelism='1', - asyncCapacity='100' -); +参数具体细节请看[命令参数说明](./config.md) -CREATE TABLE MyResult( - id bigint, - name varchar, - address varchar, - home varchar, - school varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_out', - parallelism = '1' -); - - -insert -into - MyResult - select - t1.id AS id, - t1.name AS name, - t1.address AS address, - t2.school AS school, - t2.home AS home - from - ( - select - id, - name, - address - from - MyTable - ) t1 - left join sideTable t2 - on t2.id = t2.id; -``` +任务sql详情请看[任务样例](./demo.md) -# 招聘 +### 招聘 1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 From 28085f5e8e4c59c8d5b2d3f0db4bc136956ab919 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 10:06:43 +0800 Subject: [PATCH 113/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9quickStart=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index a51f234e3..d7b613092 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -65,7 +65,7 @@ sh submit.sh # sql: 本次任务执行sql脚本 # localSqPluginPath: 本地插件包根目录地址 # remoteSqlPluginPath: flink执行集群上的插件根目录地址 -# flinkconf: flink配置文件所在目录（单机模式下不需要） +# flinkconf: flink配置文件所在目录（local模式下不需要） # yarnconf: Hadoop配置文件（包括hdfs和yarn）所在目录 # flinkJarPath: yarnPer模式提交需要指定本地的flink jar存放路径 # pluginLoadMode:yarnPer模式下的插件包加载方式 From 393c98e5cd089f58ac678866d37228e7d2d27fbe Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 16 Apr 2020 10:50:38 +0800 Subject: [PATCH 114/378] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E6=97=A5=E5=BF=97logger=E5=92=8C=E9=87=87=E7=94=A8=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E8=AE=A1=E6=95=B0=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 6 ++---- .../dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 8 ++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 7f8478183..0e59f4590 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -51,7 +51,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -65,7 +64,6 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private RuntimeContext runtimeContext; - private final static AtomicLong FAIL_NUM = new AtomicLong(0); private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; protected BaseSideInfo sideInfo; @@ -257,9 +255,9 @@ protected void cancelTimerWhenComplete(ResultFuture resultFuture, Schedule } protected void dealFillDataError(CRow input, ResultFuture resultFuture, Throwable e) { - if(FAIL_NUM.incrementAndGet() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ + parseErrorRecords.inc(); + if(parseErrorRecords.getCount() > sideInfo.getSideTableInfo().getAsyncFailMaxNum(Long.MAX_VALUE)){ LOG.info("dealFillDataError", e); - parseErrorRecords.inc(); resultFuture.completeExceptionally(e); } else { dealMissKey(input, resultFuture); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 9571d7962..b34e8297c 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -76,13 +76,10 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); - private Logger logger = LoggerFactory.getLogger(getClass()); - public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); } - @Override protected void preInvoke(CRow input, ResultFuture resultFuture){ @@ -93,7 +90,6 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul AtomicLong networkLogCounter = new AtomicLong(0L); while (!CONN_STATUS.get()){//network is unhealth - //todo:统一计数 if(networkLogCounter.getAndIncrement() % 1000 == 0){ LOG.info("network unhealth to block task"); } @@ -122,7 +118,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul try { if(conn.failed()){ if(failCounter.getAndIncrement() % 1000 == 0){ - logger.error("getConnection error", conn.cause()); + LOG.error("getConnection error", conn.cause()); } if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ dealFillDataError(input, resultFuture, conn.cause()); @@ -146,7 +142,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul try { latch.wait(); } catch (InterruptedException e) { - logger.error("", e); + LOG.error("", e); } } From d394f209f7dc44f4880066db0ed56ee0ff6eca8b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 10:55:18 +0800 Subject: [PATCH 115/378] =?UTF-8?q?quickStart=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=9B=9B=E7=A7=8D=E6=A8=A1=E5=BC=8F=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 48 +++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index d7b613092..f2dcc39ab 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -46,7 +46,27 @@ mvn clean package -Dmaven.test.skip #### 1.4.1 启动命令 ```shell script -# 脚本启动 +# local模式命令 +sh submit.sh + -mode local + -name local_test + -sql F:\dtstack\stressTest\flinkStreamSql\stressTest.sql + -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + +#standalone模式命令 +sh submit.sh + -mode standalone + -sql F:\dtstack\flinkStreamSql\tiezhu\twodimjoin.sql + -name wtz_standalone_flinkStreamSql + -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins + -remoteSqlPluginPath /home/admin/dtstack/flinkStreamSQL/plugins + -flinkconf F:\dtstack\flinkStreamSql\localhost\flinkConf + -yarnconf F:\dtstack\flinkStreamSql\localhost\hadoop + -flinkJarPath F:\Java\flink-1.8.2-bin-scala_2.12\flink-1.8.2\lib + -pluginLoadMode shipfile + -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} + +# yarn模式命令 sh submit.sh -mode yarn -name flink1.10_yarnSession @@ -57,20 +77,22 @@ sh submit.sh -yarnconf F:\dtstack\flinkStreamSql\yarnConf_node1 -flinkJarPath F:\dtstack\flink\flink-1.10.0\lib -pluginLoadMode shipfile - -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} -# mode: 任务启动的模式 -# name: 本次任务名称 -# sql: 本次任务执行sql脚本 -# localSqPluginPath: 本地插件包根目录地址 -# remoteSqlPluginPath: flink执行集群上的插件根目录地址 -# flinkconf: flink配置文件所在目录（local模式下不需要） -# yarnconf: Hadoop配置文件（包括hdfs和yarn）所在目录 -# flinkJarPath: yarnPer模式提交需要指定本地的flink jar存放路径 -# pluginLoadMode:yarnPer模式下的插件包加载方式 -# confProp: 其他额外参数配置 -# yarnSessionConf: yarnSession模式下指定的运行参数，目前只支持指定yid +# yarnPer模式命令 +sh submit.sh + -mode yarnPer + -sql /home/wen/Desktop/flink_stream_sql_conf/sql/Test01.sql + -name TestAll + -localSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins + -remoteSqlPluginPath /home/wen/IdeaProjects/flinkStreamSQL/plugins + -flinkconf /home/wen/Desktop/flink_stream_sql_conf/flinkConf + -yarnconf /home/wen/Desktop/flink_stream_sql_conf/yarnConf_node1 + -flinkJarPath /home/wen/Desktop/dtstack/flink-1.8.1/lib + -pluginLoadMode shipfile + -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} + -queue c + ``` 参数具体细节请看[命令参数说明](./config.md) From d68751dfe9422d1f659256396663b6eb1ef8f3c0 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 11:00:06 +0800 Subject: [PATCH 116/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9quickStart=E6=8E=92?= =?UTF-8?q?=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/quickStart.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/quickStart.md b/docs/quickStart.md index f2dcc39ab..a5bf1fa9a 100644 --- a/docs/quickStart.md +++ b/docs/quickStart.md @@ -23,6 +23,7 @@ zookeeper.sasl.login-context-name: Client ``` ### 1.3 打包 + 进入项目根目录，使用maven打包： ```shell script mvn clean package -Dmaven.test.skip @@ -41,19 +42,19 @@ mvn clean package -Dmaven.test.skip | |--- xxxsink | |--- xxxside ``` -### 1.4 快速启动 - -#### 1.4.1 启动命令 +### 1.4 快速启动命令 +#### local模式命令 ```shell script -# local模式命令 sh submit.sh -mode local -name local_test -sql F:\dtstack\stressTest\flinkStreamSql\stressTest.sql -localSqlPluginPath F:\dtstack\project\flinkStreamSQL\plugins +``` -#standalone模式命令 +#### standalone模式命令 +```shell script sh submit.sh -mode standalone -sql F:\dtstack\flinkStreamSql\tiezhu\twodimjoin.sql @@ -65,8 +66,10 @@ sh submit.sh -flinkJarPath F:\Java\flink-1.8.2-bin-scala_2.12\flink-1.8.2\lib -pluginLoadMode shipfile -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} +``` -# yarn模式命令 +#### yarn模式命令 +```shell script sh submit.sh -mode yarn -name flink1.10_yarnSession @@ -78,8 +81,10 @@ sh submit.sh -flinkJarPath F:\dtstack\flink\flink-1.10.0\lib -pluginLoadMode shipfile -yarnSessionConf {\"yid\":\"application_1586851105774_0014\"} +``` -# yarnPer模式命令 +#### yarnPer模式命令 +```shell script sh submit.sh -mode yarnPer -sql /home/wen/Desktop/flink_stream_sql_conf/sql/Test01.sql @@ -92,7 +97,6 @@ sh submit.sh -pluginLoadMode shipfile -confProp {\"time.characteristic\":\"eventTime\",\"logLevel\":\"info\"} -queue c - ``` 参数具体细节请看[命令参数说明](./config.md) From 92baf834cd0a75230e5b5ff11fc5e0456b496070 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 11:56:54 +0800 Subject: [PATCH 117/378] add rdbSideParams.md --- docs/plugin/rdbSideParams.md | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/plugin/rdbSideParams.md diff --git a/docs/plugin/rdbSideParams.md b/docs/plugin/rdbSideParams.md new file mode 100644 index 000000000..26666bd5b --- /dev/null +++ b/docs/plugin/rdbSideParams.md @@ -0,0 +1,41 @@ +## 关系型数据库维表参数 + +适用于`MYSQL`,`ORACLE`,`SQLSERVER`,`POSTGRESQL`,`DB2`,`POLARDB`,`CLICKHOUSE`,`IMPALA`维表插件 + +### 维表参数 + +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +| type | 维表类型， 例如:mysql |是|| +| url | 连接数据库 jdbcUrl |是|| +| userName | 连接用户名 |是|| +| password | 连接密码|是|| +| tableName| 表名称|是|| +| schema| 表空间|否|| +| cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU| +| partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| + +### 缓存策略 + +- NONE：不做内存缓存。每条流数据触发一次维表查询操作。 +- ALL: 任务启动时，一次性加载所有数据到内存，并进行缓存。适用于维表数据量较小的情况。 +- LRU: 任务执行时，根据维表关联条件使用异步算子加载维表数据，并进行缓存。 + +#### ALL全量维表参数 + +|参数名称|含义|默认值| +|----|---|----| +| cacheTTLMs | 缓存周期刷新时间 |60，单位s| + +#### LRU异步维表参数 + +|参数名称|含义|默认值| +|----|---|----| +| cacheTTLMs | LRU缓存写入后超时时间 |60，单位s| +| cacheSize | LRU缓存大小 |10000| +| cacheMode | 异步请求处理有序还是无序，可选：ordered，unordered |ordered| +| asyncCapacity | 异步线程容量 |100| +| asyncTimeout | 异步处理超时时间 |10000，单位毫秒| +| asyncPoolSize | 异步查询DB最大线程池，上限20 |min(20,Runtime.getRuntime().availableProcessors() * 2)| + + From 4566ece8e09a97af3e16e773f8298e018007b062 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 14:27:11 +0800 Subject: [PATCH 118/378] oracle mysql side table doc --- docs/plugin/mysqlSide.md | 149 ++++++++++++++++++++++++----------- docs/plugin/oracleSide.md | 133 +++++++++++++++++++++++-------- docs/plugin/rdbSideParams.md | 1 + 3 files changed, 203 insertions(+), 80 deletions(-) diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index 6a0e0aea9..96e997e9e 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -1,11 +1,15 @@ ## 1.格式： + + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + ``` CREATE TABLE tableName( colName cloType, ... PRIMARY KEY(keyInfo), - PERIOD FOR SYSTEM_TIME + PERIOD FOR SYSTEM_TIME )WITH( type='mysql', url='jdbcUrl', @@ -24,62 +28,117 @@ mysql-5.6.35 ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | mysql表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| + + [关系型数据库维表参数](docs/rdbSideParams.md) -## 4.参数 +## 4.样例 - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 mysql |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | mysql连接用户名 |是|| - | password | mysql连接密码|是|| - | tableName | mysql表名称|是|| - | tableName | mysql 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 +### ALL全量维表定义 +``` + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='test_mysql_10', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='2' + ); -## 5.样例 ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), +### LRU异步维表定义 + +``` +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='yctest_mysql_10', + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' - parallelism ='1', - partitionedJoin='false' + asyncPoolSize ='3', + parallelism ='2' ); +``` + +### MySQL异步维表关联 ``` +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='yctest_mysql_mq', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.204:3306/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='yctest_mysql_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; +``` \ No newline at end of file diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index 47b8e36c7..1cb4e4cce 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -1,5 +1,13 @@ ## 1.格式： + + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + + 此外, 针对oracle char类型自动补齐的特性，我们允许为char指定长度，确保维表查询时能够匹配数据。 + + + 注意：Oracle维表使用的字段大小写，需要和Oracle中定义的保持一致。 ``` CREATE TABLE tableName( colName cloType, @@ -22,43 +30,35 @@ # 2.支持版本 10g 11g -## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | oracle表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 oracle |是|| - | url | 连接oracle数据库 jdbcUrl |是|| - | userName | oracle连接用户名 |是|| - | password | oracle连接密码|是|| - | tableName | oracle表名称|是|| - | schema | oracle 的schema|否|当前登录用户| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 - -## 5.样例 +## 3. 表结构定义 + [关系型数据库维表参数](docs/rdbSideParams.md) + +## 4.样例 + +### ALL全量维表定义 +``` +CREATE TABLE sideTable( + ID char(20), // oracle定义了char(20) + NAME varchar, + PRIMARY KEY (ID), + PERIOD FOR SYSTEM_TIME + )WITH( + type='oracle', + url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe', + userName = 'system', + password = 'oracle', + tableName = 'SIDETEST1', + schema = 'dtstack', + cache = 'ALL', + cacheTTLMs ='60000' + ); +``` + +### LRU异步维表定义 + ``` create table sideTable( - channel varchar, + channel char, xccount int, PRIMARY KEY(channel), PERIOD FOR SYSTEM_TIME @@ -79,7 +79,70 @@ create table sideTable( schema = 'MQTEST' ); +``` + +### Oracle异步维表关联 + +```aidl +CREATE TABLE MyTable( + id varchar, + name varchar + --ts timestamp, + --tsDate Date + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + topicIsPattern ='false', + parallelism ='1' + ); + + +CREATE TABLE sideTable( + ID char(20), + NAME varchar, + PRIMARY KEY (ID), + PERIOD FOR SYSTEM_TIME + )WITH( + type='oracle', + url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe', + userName = 'system', + password = 'oracle', + tableName = 'SIDETEST1', + --schema = 'dtstack', + cache = 'LRU', + asyncPoolSize ='3' + ); + + +CREATE TABLE MyResult( + NAME varchar, + ID char(20), + PRIMARY KEY (ID) + )WITH( + --type ='console', + type='oracle', + url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe', + userName = 'system', + password = 'oracle', + tableName = 'SINK_TEST', + batchSize='1' + ); + + +INSERT INTO MyResult +SELECT + s.ID as ID, + m.name as NAME +FROM MyTable m +LEFT JOIN + sideTable s +ON + m.id=s.ID ``` diff --git a/docs/plugin/rdbSideParams.md b/docs/plugin/rdbSideParams.md index 26666bd5b..34d02978e 100644 --- a/docs/plugin/rdbSideParams.md +++ b/docs/plugin/rdbSideParams.md @@ -14,6 +14,7 @@ | schema| 表空间|否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU| | partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| +| parallelism | 处理后的数据流并行度|否|| ### 缓存策略 From b264114417d7044274972d3b3309abc612a02d30 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 14:30:45 +0800 Subject: [PATCH 119/378] link rdbSideParams.md --- docs/plugin/mysqlSide.md | 2 +- docs/plugin/oracleSide.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index 96e997e9e..f28f09518 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [关系型数据库维表参数](docs/rdbSideParams.md) + [关系型数据库维表参数](docs/plugin/rdbSideParams.md) ## 4.样例 diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index 1cb4e4cce..ee7e03331 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -31,7 +31,7 @@ 10g 11g ## 3. 表结构定义 - [关系型数据库维表参数](docs/rdbSideParams.md) + [关系型数据库维表参数](docs/plugin/rdbSideParams.md) ## 4.样例 From 1e2db49880757b2820544e0bf9a91081652fc95a Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 15:52:06 +0800 Subject: [PATCH 120/378] =?UTF-8?q?update=20redis=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 2 +- docs/plugin/redisSide.md | 46 +++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/config.md b/docs/config.md index 3fb6ac723..9aa8df994 100644 --- a/docs/config.md +++ b/docs/config.md @@ -73,7 +73,7 @@ sh submit.sh -key1 val1 -key2 val2 * savePointPath：任务恢复点的路径（默认无） * allowNonRestoredState：指示保存点是否允许非还原状态的标志（默认false） * logLevel: 日志级别动态配置（默认info） - * [prometheus 相关参数](docs/prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 + * [prometheus 相关参数](./prometheus.md) per_job可指定metric写入到外部监控组件,以prometheus pushgateway举例 * **flinkconf** diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index f9b5e41f9..6aa335fea 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -22,29 +22,31 @@ redis5.0 ## 3.表结构定义 +注意：redis中没有表和schema的概念，参数中tableName是指符合命名规则的key，具体规则请看[缓存redis的存储命名规则] |参数名称|含义| |----|---| -| tableName | 注册到flink的表名称(可选填;不填默认和redis对应的表名称相同)| -| colName | 列名称，维表列名格式 表名:主键名:主键值:列名]| +| tableName | 注册到flinkStreamSql的表名称(可选填;不填默认和redis对应的"表"名称相同)| +| colName | 列名称，对应redis对应"表"的field| | colType | 列类型，当前只支持varchar| -| PRIMARY KEY |主键，多个字段做为联合主键时以逗号分隔 -| PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| +| PRIMARY KEY |主键，多个字段做为联合主键时以逗号分隔| +| PERIOD FOR SYSTEM_TIME | 关键字，表明该定义的表为维表信息| ## 4.参数 +参数详细说明请看[参数详细说明]() + |参数名称|含义|是否必填|默认值| |----|---|---|----| | type | 表明维表的类型[hbase|mysql|redis]|是|| | url | redis 的地址;格式ip:port[,ip:port]|是|| -| password | redis 的密码 |是|| +| password | redis 的密码 |否|空| | redisType | redis模式（1 单机，2 哨兵， 3 集群）| 是 | | masterName | 主节点名称（哨兵模式下为必填项） | 否 | -| database | reids 的数据库地址|否|| -| tableName | redis 的表名称|是|| +| database | reids 的数据库地址|否|0| +| tableName | redis 的“表”名称|是|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - -------------- > 缓存策略 * NONE: 不做内存缓存 @@ -55,27 +57,37 @@ ## 5.样例 ``` -create table sideTable( - channel varchar, - xccount varchar, - PRIMARY KEY(channel), +CREATE TABLE MyRedis( + id varchar, + message varchar, + PRIMARY KEY(id), PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( type='redis', url='172.16.10.79:6379', password='abc123', database='0', redisType = '1', - tableName='sidetest', + tableName = 'sideTable', cache = 'LRU', cacheTTLMs='10000' - ); +); ``` -## 6.缓存redis的存储结构规则 -``` +## 6.redis的存储命名规则 + redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} 如果以班级class表为例，id和name作为联合主键，那么redis的结构为 + +在样例中，tableName为sideTable，主键为id，column为id，message，所以对应的redis数据插入语句为 + +数据在redis中对应的数据存储情况为： +``` +192.168.80.105:6379> hgetall sideTable_5 +1) "id" +2) "5" +3) "message" +4) "redis" ``` From bbf37e1df45ee7f589522177dc5f3eabdfe5dbc5 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 16 Apr 2020 16:05:08 +0800 Subject: [PATCH 121/378] oracle mysql side doc --- docs/plugin/mysqlSide.md | 14 +++++++++++++- docs/plugin/oracleSide.md | 13 ++++++++++++- docs/plugin/{rdbSideParams.md => sideParams.md} | 11 ++++------- 3 files changed, 29 insertions(+), 9 deletions(-) rename docs/plugin/{rdbSideParams.md => sideParams.md} (76%) diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index f28f09518..834183ed2 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -29,7 +29,19 @@ ## 3.表结构定义 - [关系型数据库维表参数](docs/plugin/rdbSideParams.md) + [维表参数信息](docs/plugin/sideParams.md) + + mysql独有的参数配置： + + +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +| type | 维表类型， mysql |是|| +| url | 连接数据库 jdbcUrl |是|| +| userName | 连接用户名 |是|| +| password | 连接密码|是|| + + ## 4.样例 diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index ee7e03331..b681536f8 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -31,7 +31,18 @@ 10g 11g ## 3. 表结构定义 - [关系型数据库维表参数](docs/plugin/rdbSideParams.md) + [维表参数](docs/plugin/sideParams.md) + + + oracle独有的参数配置： + + |参数名称|含义|是否必填|默认值| + |----|---|---|----| + | type | 维表类型， oracle |是|| + | url | 连接数据库 jdbcUrl |是|| + | userName | 连接用户名 |是|| + | password | 连接密码|是|| + | schema| 表空间|否|| ## 4.样例 diff --git a/docs/plugin/rdbSideParams.md b/docs/plugin/sideParams.md similarity index 76% rename from docs/plugin/rdbSideParams.md rename to docs/plugin/sideParams.md index 34d02978e..6059dcab8 100644 --- a/docs/plugin/rdbSideParams.md +++ b/docs/plugin/sideParams.md @@ -1,17 +1,14 @@ -## 关系型数据库维表参数 +## 维表参数 -适用于`MYSQL`,`ORACLE`,`SQLSERVER`,`POSTGRESQL`,`DB2`,`POLARDB`,`CLICKHOUSE`,`IMPALA`维表插件 ### 维表参数 +维表需要的基本属性，每个插件还需要提供连接所需的基本信息。 + |参数名称|含义|是否必填|默认值| |----|---|---|----| | type | 维表类型， 例如:mysql |是|| -| url | 连接数据库 jdbcUrl |是|| -| userName | 连接用户名 |是|| -| password | 连接密码|是|| | tableName| 表名称|是|| -| schema| 表空间|否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU| | partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| | parallelism | 处理后的数据流并行度|否|| @@ -37,6 +34,6 @@ | cacheMode | 异步请求处理有序还是无序，可选：ordered，unordered |ordered| | asyncCapacity | 异步线程容量 |100| | asyncTimeout | 异步处理超时时间 |10000，单位毫秒| -| asyncPoolSize | 异步查询DB最大线程池，上限20 |min(20,Runtime.getRuntime().availableProcessors() * 2)| +| asyncPoolSize | 异步查询DB最大线程池，上限20。适用于MYSQL,ORACLE,SQLSERVER,POSTGRESQL,DB2,POLARDB,CLICKHOUSE,IMPALA维表插件|min(20,Runtime.getRuntime().availableProcessors() * 2)| From dd7987e420d5f4d52008491acd8edfb3c63ab4e5 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 16 Apr 2020 16:21:14 +0800 Subject: [PATCH 122/378] =?UTF-8?q?redisSide=20=E6=B7=BB=E5=8A=A0=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSide.md | 99 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index 6aa335fea..3f4483e65 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -1,5 +1,5 @@ -## 1.格式： +## 1.格式 ``` CREATE TABLE tableName( colName cloType, @@ -15,7 +15,9 @@ redisType = '1', cache ='LRU', cacheSize ='10000', - cacheTTLMs ='60000' + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' ); ``` ## 2.支持版本 @@ -34,7 +36,7 @@ ## 4.参数 -参数详细说明请看[参数详细说明]() +参数详细说明请看[参数详细说明](./sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -48,14 +50,9 @@ | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| -------------- -> 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * ALL: 缓存全量表数据 ## 5.样例 +### LRU维表示例 ``` CREATE TABLE MyRedis( id varchar, @@ -70,10 +67,92 @@ CREATE TABLE MyRedis( redisType = '1', tableName = 'sideTable', cache = 'LRU', - cacheTTLMs='10000' + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +``` +### ALL维表示例 +``` +CREATE TABLE MyRedis( + id varchar, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='redis', + url='172.16.10.79:6379', + password='abc123', + database='0', + redisType = '1', + tableName = 'sideTable', + cache = 'ALL', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +``` +### redis异步维表关联完整案例 +``` +CREATE TABLE MyTable( + id varchar, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' ); +CREATE TABLE MyRedis( + id varchar, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='redis', + url='172.16.10.79:6379', + password='abc123', + database='0', + redisType = '1', + tableName = 'sideTable', + cache = 'LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); + +CREATE TABLE MyResult( + id varchar, + name varchar, + address varchar, + message varchar +)WITH( + type = 'console' +); + +insert into MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from + ( + select id, name, address + from MyTable + ) t1 join MyRedis t2 + on t1.id = t2.id; ``` + ## 6.redis的存储命名规则 redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} From 38e34dc546ee421530915d5c2b387304ac6b8fdf Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 17 Apr 2020 11:06:19 +0800 Subject: [PATCH 123/378] =?UTF-8?q?update=20hbase=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 140 ++++++++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 23 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 0c4e545f9..3d08084d0 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -1,5 +1,5 @@ -## 1.格式： +## 1.格式 ``` CREATE TABLE tableName( columnFamily:columnName type as alias, @@ -31,7 +31,9 @@ | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;hbase 维表rowkey的构造方式;可选择的构造包括 md5(alias + alias), '常量',也包括上述方式的自由组合 | -## 3.参数 +## 4.参数 + +参数详细说明请看[参数详细说明](./sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -43,31 +45,123 @@ | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| -------------- -> 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) -## 4.样例 +## 5.样例 +### LRU维表示例 +``` +CREATE TABLE sideTable ( + wtz:message varchar as message, + wtz:info varchar as info , + PRIMARY KEY (rowkey), + PERIOD FOR SYSTEM_TIME +) WITH ( + type = 'hbase', + zookeeperQuorum = '192.168.80.105:2181,192.168.80.106:2181,192.168.80.107:2181', + zookeeperParent = '/hbase', + tableName = 'testFlinkStreamSql', + parallelism = '1', + cache = 'LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +``` + +### ALL维表示例 +``` +CREATE TABLE sideTable ( + wtz:message varchar as message, + wtz:info varchar as info , + PRIMARY KEY (rowkey), + PERIOD FOR SYSTEM_TIME +) WITH ( + type = 'hbase', + zookeeperQuorum = '192.168.80.105:2181,192.168.80.106:2181,192.168.80.107:2181', + zookeeperParent = '/hbase', + tableName = 'testFlinkStreamSql', + parallelism = '1', + cache = 'ALL', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); ``` -CREATE TABLE sideTable( - cf:name varchar as name, - cf:info int as info, - PRIMARY KEY(md5(name) + 'test'), - PERIOD FOR SYSTEM_TIME - )WITH( - type ='hbase', - zookeeperQuorum ='rdos1:2181', - zookeeperParent ='/hbase', - tableName ='workerinfo', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='true' + +### hbase异步维表关联完整案例 +``` +CREATE TABLE MyTable( + id varchar, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE MyResult( + id varchar, + name varchar, + address varchar, + message varchar, + info varchar +)WITH( + type = 'console' ); + + CREATE TABLE sideTable ( + wtz:message varchar as message, + wtz:info varchar as info , + PRIMARY KEY (rowkey), + PERIOD FOR SYSTEM_TIME +) WITH ( + type = 'hbase', + zookeeperQuorum = '192.168.80.105:2181,192.168.80.106:2181,192.168.80.107:2181', + zookeeperParent = '/hbase', + tableName = 'testFlinkStreamSql', + parallelism = '1', + cache = 'LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + partitionedJoin='false' +); +insert +into + MyResult + select + a.name, + a.id, + a.address, + b.message, + b.info + from + MyTable a + left join + sideTable b + on a.id=b.rowkey; ``` +## 6.hbase中数据存储形式 + +在hbase中，数据是以列簇的形式存储，其中rowKey作为主键，按字典排序。 +在样例中，wtz为列族名，message, info为列名，数据在hbase中的存储情况为： +``` +hbase(main):002:0> scan 'testFlinkStreamSql' +ROW COLUMN+CELL + 0 column=wtz:info, timestamp=1587089266719, value=hadoop + 0 column=wtz:message, timestamp=1587089245780, value=hbase + 1 column=wtz:info, timestamp=1587088818432, value=flink + 1 column=wtz:message, timestamp=1587088796633, value=dtstack + 2 column=wtz:info, timestamp=1587088858564, value=sql + 2 column=wtz:message, timestamp=1587088840507, value=stream +``` From 774da5c35d3abc07abf90aa9494d44531013b98b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 17 Apr 2020 11:07:57 +0800 Subject: [PATCH 124/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9hbase=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E6=8E=92=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 3d08084d0..2ab50f914 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -107,13 +107,13 @@ CREATE TABLE MyTable( ); CREATE TABLE MyResult( - id varchar, - name varchar, - address varchar, - message varchar, - info varchar + id varchar, + name varchar, + address varchar, + message varchar, + info varchar )WITH( - type = 'console' + type = 'console' ); CREATE TABLE sideTable ( From 4bbe75178dc71115df4236078078b9d7526199f8 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 17 Apr 2020 11:36:25 +0800 Subject: [PATCH 125/378] =?UTF-8?q?update=20hbase,=20redis=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E6=8E=92=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 33 +++++++++-------- docs/plugin/redisSide.md | 77 ++++++++++++++++++++-------------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 2ab50f914..cc5fa319c 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -88,6 +88,24 @@ CREATE TABLE sideTable ( ); ``` +## 6.hbase完整样例 + +### 数据说明 + +在hbase中，数据是以列簇的形式存储，其中rowKey作为主键，按字典排序。 + +在样例中，wtz为列族名，message, info为列名，数据在hbase中的存储情况为： +``` +hbase(main):002:0> scan 'testFlinkStreamSql' +ROW COLUMN+CELL + 0 column=wtz:info, timestamp=1587089266719, value=hadoop + 0 column=wtz:message, timestamp=1587089245780, value=hbase + 1 column=wtz:info, timestamp=1587088818432, value=flink + 1 column=wtz:message, timestamp=1587088796633, value=dtstack + 2 column=wtz:info, timestamp=1587088858564, value=sql + 2 column=wtz:message, timestamp=1587088840507, value=stream +``` + ### hbase异步维表关联完整案例 ``` CREATE TABLE MyTable( @@ -150,18 +168,3 @@ into on a.id=b.rowkey; ``` -## 6.hbase中数据存储形式 - -在hbase中，数据是以列簇的形式存储，其中rowKey作为主键，按字典排序。 - -在样例中，wtz为列族名，message, info为列名，数据在hbase中的存储情况为： -``` -hbase(main):002:0> scan 'testFlinkStreamSql' -ROW COLUMN+CELL - 0 column=wtz:info, timestamp=1587089266719, value=hadoop - 0 column=wtz:message, timestamp=1587089245780, value=hbase - 1 column=wtz:info, timestamp=1587088818432, value=flink - 1 column=wtz:message, timestamp=1587088796633, value=dtstack - 2 column=wtz:info, timestamp=1587088858564, value=sql - 2 column=wtz:message, timestamp=1587088840507, value=stream -``` diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index 3f4483e65..c63c542cd 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -93,6 +93,25 @@ CREATE TABLE MyRedis( partitionedJoin='false' ); ``` + +## 6.redis完整样例 + +### redis数据说明 + +redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} +如果以班级class表为例，id和name作为联合主键，那么redis的结构为 + +在样例中，tableName为sideTable，主键为id，column为id，message，所以对应的redis数据插入语句为 + +数据在redis中对应的数据存储情况为： +``` +192.168.80.105:6379> hgetall sideTable_5 +1) "id" +2) "5" +3) "message" +4) "redis" +``` + ### redis异步维表关联完整案例 ``` CREATE TABLE MyTable( @@ -112,12 +131,12 @@ CREATE TABLE MyTable( ); CREATE TABLE MyRedis( - id varchar, - message varchar, - PRIMARY KEY(id), - PERIOD FOR SYSTEM_TIME + id varchar, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME )WITH( - type='redis', + type='redis', url='172.16.10.79:6379', password='abc123', database='0', @@ -131,42 +150,24 @@ CREATE TABLE MyRedis( ); CREATE TABLE MyResult( - id varchar, - name varchar, - address varchar, - message varchar + id varchar, + name varchar, + address varchar, + message varchar )WITH( - type = 'console' + type = 'console' ); insert into MyResult - select - t1.id AS id, - t1.name AS name, - t1.address AS address, - t2.message AS message - from - ( - select id, name, address - from MyTable - ) t1 join MyRedis t2 - on t1.id = t2.id; -``` - -## 6.redis的存储命名规则 - -redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} -如果以班级class表为例，id和name作为联合主键，那么redis的结构为 - -在样例中，tableName为sideTable，主键为id，column为id，message，所以对应的redis数据插入语句为 - -数据在redis中对应的数据存储情况为： + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from( + select id, name, address + from MyTable + ) t1 join MyRedis t2 + on t1.id = t2.id; ``` -192.168.80.105:6379> hgetall sideTable_5 -1) "id" -2) "5" -3) "message" -4) "redis" -``` - From f919e35185846ee4330882d0f97e3375f2055503 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Apr 2020 11:27:10 +0800 Subject: [PATCH 126/378] =?UTF-8?q?add=20isHighAvailability=20=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/launcher/ClusterClientFactory.java | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 14cd847b5..eb06b39ba 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -24,15 +24,12 @@ import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; @@ -57,6 +54,18 @@ */ public class ClusterClientFactory { + private static final Logger LOG = LoggerFactory.getLogger(ClusterClientFactory.class); + + private static final String HA_CLUSTER_ID = "high-availability.cluster-id"; + + private static final String HIGH_AVAILABILITY = "high-availability"; + + private static final String NODE = "NONE"; + + private static final String ZOOKEEPER = "zookeeper"; + + private static final String HADOOP_CONF = "fs.hdfs.hadoopconf"; + public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if (mode.equals(ClusterMode.standalone.name())) { @@ -70,10 +79,12 @@ public static ClusterClient createClusterClient(Options launcherOptions) throws public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(config); - MiniCluster miniCluster = new MiniCluster(configBuilder.build()); - MiniClusterClient clusterClient = new MiniClusterClient(config, miniCluster); + + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + + RestClusterClient clusterClient = new RestClusterClient<>(config, "clusterClient"); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); config.setString(JobManagerOptions.ADDRESS, address.getAddress().getHostName()); @@ -89,6 +100,8 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { + boolean isHighAvailability; + config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); FileSystem.initialize(config); @@ -101,6 +114,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { String yarnSessionConf = launcherOptions.getYarnSessionConf(); yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); + Object yid = yarnSessionConfProperties.get("yid"); if (null != yid) { @@ -109,12 +123,22 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - System.out.println("applicationId=" + applicationId.toString()); + LOG.info("current applicationId = {}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } + isHighAvailability = config.getString(HIGH_AVAILABILITY, NODE).equals(ZOOKEEPER); + + if (isHighAvailability && config.getString(HA_CLUSTER_ID, null) == null) { + config.setString(HA_CLUSTER_ID, applicationId.toString()); + } + + LOG.info("------------config params-------------------------"); + config.toMap().forEach((key, value) -> LOG.info("{}: {}", key, value)); + LOG.info("-------------------------------------------"); + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, flinkConfDir, yarnClient, false); ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); clusterClient.setDetached(true); @@ -158,7 +182,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) } - if (StringUtils.isEmpty(applicationId.toString())) { + if (applicationId == null || StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); } return applicationId; @@ -166,7 +190,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { + if (!"application".equals(it.next())) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { From e438f544bf0a82de74ad665abca61b95c8fddfcf Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 10 Apr 2020 09:34:55 +0800 Subject: [PATCH 127/378] fix --- .../flink/sql/launcher/LauncherMain.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index c098bc27e..cb1fffe01 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.launcher; @@ -66,8 +66,7 @@ public class LauncherMain { private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { String jarPath = PluginUtil.getCoreJarFileName(localSqlRootJar, CORE_JAR); - String corePath = localSqlRootJar + SP + jarPath; - return corePath; + return localSqlRootJar + SP + jarPath; } public static void main(String[] args) throws Exception { @@ -104,6 +103,7 @@ public static void main(String[] args) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = StringUtils.isEmpty(flinkConfDir) ? new Configuration() : GlobalConfiguration.loadConfiguration(flinkConfDir); JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, config, 1); + PerJobSubmitter.submit(launcherOptions, jobGraph, config); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); @@ -115,20 +115,20 @@ public static void main(String[] args) throws Exception { private static String[] parseJson(String[] args) { BufferedReader reader = null; - String lastStr = ""; - try{ + StringBuilder lastStr = new StringBuilder(); + try { FileInputStream fileInputStream = new FileInputStream(args[0]); - InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, Charsets.UTF_8); reader = new BufferedReader(inputStreamReader); - String tempString = null; - while((tempString = reader.readLine()) != null){ - lastStr += tempString; + String tempString; + while ((tempString = reader.readLine()) != null) { + lastStr.append(tempString); } reader.close(); - }catch(IOException e){ + } catch (IOException e) { e.printStackTrace(); - }finally{ - if(reader != null){ + } finally { + if (reader != null) { try { reader.close(); } catch (IOException e) { @@ -136,14 +136,14 @@ private static String[] parseJson(String[] args) { } } } - Map map = JSON.parseObject(lastStr, new TypeReference>(){} ); + Map map = JSON.parseObject(lastStr.toString(), new TypeReference>() { + }); List list = new LinkedList<>(); for (Map.Entry entry : map.entrySet()) { list.add("-" + entry.getKey()); list.add(entry.getValue().toString()); } - String[] array = list.toArray(new String[list.size()]); - return array; + return list.toArray(new String[0]); } } From 83429b26199feac0f30a5ec15c2e230c670a4b78 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 20 Apr 2020 19:50:17 +0800 Subject: [PATCH 128/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E8=BF=87=E7=A8=8B=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/launcher/ClusterClientFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index eb06b39ba..cfbab1023 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -25,6 +25,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -39,6 +40,8 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.StringHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.net.URLDecoder; From 38b9b388b6b517631080d1f17135a175528bfe54 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 21 Apr 2020 11:21:46 +0800 Subject: [PATCH 129/378] =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E8=A1=A8=E5=90=8D?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/util/FieldReplaceUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index f92643d5f..69bdc97b2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -181,12 +181,12 @@ public static SqlIdentifier createNewIdentify(SqlIdentifier sqlIdentifier, return sqlIdentifier; } + sqlIdentifier = sqlIdentifier.setName(0, newTbName); + String mappingFieldName = mappingField.get(fieldName); if(mappingFieldName == null){ return sqlIdentifier; } - - sqlIdentifier = sqlIdentifier.setName(0, newTbName); sqlIdentifier = sqlIdentifier.setName(1, mappingFieldName); return sqlIdentifier; } From ca8d08de02594d5abfe8efa6cc687219e0a274ad Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 21 Apr 2020 12:53:24 +0800 Subject: [PATCH 130/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcase=20when=20?= =?UTF-8?q?=E4=B8=AD=E6=B2=A1=E6=9C=89=E8=A7=A3=E6=9E=90else=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/JoinNodeDealer.java | 2 +- .../java/com/dtstack/flink/sql/util/FieldReplaceUtil.java | 4 ++++ .../main/java/com/dtstack/flink/sql/util/TableUtils.java | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index f072e2591..544e8ebb0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -233,7 +233,6 @@ public Set extractField(SqlNode sqlNode, Set fieldFromJoinCondition = extractSelectFieldFromJoinCondition(joinFieldSet, fromTableNameSet, tableRef); Set extractGroupByField = extractFieldFromGroupByList(parentGroupByList, fromTableNameSet, tableRef); - extractSelectField.addAll(extractCondition); extractSelectField.addAll(fieldFromJoinCondition); extractSelectField.addAll(extractGroupByField); @@ -575,6 +574,7 @@ private void extractSelectField(SqlNode selectNode, || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java index 69bdc97b2..775e0a985 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FieldReplaceUtil.java @@ -160,6 +160,10 @@ private static SqlNode replaceNodeInfo(SqlNode groupNode, SqlNode replaceNode = replaceSelectFieldName(sqlNode, oldTbName, newTbName, mappingField); sqlCase.getThenOperands().set(i,replaceNode); } + if(sqlCase.getElseOperand() != null){ + SqlNode replaceNode = replaceSelectFieldName(sqlCase.getElseOperand(), oldTbName, newTbName, mappingField); + sqlCase.setOperand(3, replaceNode); + } return sqlCase; } else { return groupNode; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 579beaf63..4a4973901 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -151,6 +151,7 @@ public static void extractSelectFieldToFieldInfo(SqlNode fieldNode, String fromN || fieldNode.getKind() == TIMESTAMP_ADD || fieldNode.getKind() == TIMESTAMP_DIFF || fieldNode.getKind() == LIKE + || fieldNode.getKind() == COALESCE ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) fieldNode; for (int i = 0; i < sqlBasicCall.getOperands().length; i++) { @@ -363,6 +364,7 @@ public static void replaceSelectFieldTable(SqlNode selectNode, || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; @@ -548,6 +550,7 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ){ SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; @@ -573,6 +576,7 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S sqlCase.getWhenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); sqlCase.getThenOperands().getList().forEach(sqlNode -> replaceConditionNode(sqlNode, oldTbName, newTbName, fieldReplaceRef)); + replaceConditionNode(sqlCase.getElseOperand(), oldTbName, newTbName, fieldReplaceRef); } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } @@ -620,6 +624,7 @@ public static void getConditionRefTable(SqlNode selectNode, Set fieldInf || selectNode.getKind() == TIMESTAMP_ADD || selectNode.getKind() == TIMESTAMP_DIFF || selectNode.getKind() == LIKE + || selectNode.getKind() == COALESCE ) { SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; @@ -645,6 +650,7 @@ public static void getConditionRefTable(SqlNode selectNode, Set fieldInf sqlCase.getWhenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); sqlCase.getThenOperands().getList().forEach(sqlNode -> getConditionRefTable(sqlNode, fieldInfos)); + getConditionRefTable(sqlCase.getElseOperand(), fieldInfos); } else { throw new RuntimeException(String.format("not support node kind of %s to replace name now.", selectNode.getKind())); } From b231974e816aafa5d8d685921e1404b7a1211249 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 21 Apr 2020 16:06:35 +0800 Subject: [PATCH 131/378] fix --- docs/plugin/mysqlSink.md | 73 +++++++++++++++++++++++++++++++++------- docs/plugin/redisSink.md | 5 +++ 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index 300086e21..8dabea047 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -22,7 +22,7 @@ CREATE TABLE tableName( |参数名称|含义| |----|---| -| tableName| mysql表名称| +| tableName| 结果表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| @@ -33,21 +33,70 @@ CREATE TABLE tableName( |type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| |url | 连接mysql数据库 jdbcUrl |是|| |userName | mysql连接用户名 |是|| -| password | mysql连接密码|是|| -| tableName | mysql表名称|是|| -| parallelism | 并行度设置|否|1| - -## 5.样例： +|password | mysql连接密码|是|| +|tableName | mysql表名称|是|| +|parallelism | 并行度设置|否|1| +|batchNum | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| + +## 5.完整样例： ``` +CREATE TABLE MyTable( + id int, + channel varchar, + pv varchar, + xctime varchar, + name varchar + )WITH( + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR + pv VARCHAR, + channel VARCHAR )WITH( type ='mysql', - url ='jdbc:mysql://172.16.8.104:3306/test?charset=utf8', + url ='jdbc:mysql://172.16.10.134:3306/test', userName ='dtstack', password ='abc123', - tableName ='pv2', + tableName ='myresult', parallelism ='1' - ) - ``` \ No newline at end of file + ); + + +insert +into + MyResult + select + channel, + pv + from + MyTable + ``` + +## 6.结果表数据示例： +``` +mysql> desc myresult; ++---------+--------------+------+-----+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++---------+--------------+------+-----+---------+-------+ +| channel | varchar(255) | YES | | NULL | | +| pv | varchar(11) | YES | | NULL | | ++---------+--------------+------+-----+---------+-------+ +2 rows in set (0.00 sec) + +mysql> select * from myresult limit 1; ++---------+------+ +| channel | pv | ++---------+------+ +| aa | mq6 | +``` \ No newline at end of file diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index 6a754e5c6..b43cb7377 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -41,6 +41,11 @@ redis5.0 | database | reids 的数据库地址|否|| | tableName | redis 的表名称|是|| | parallelism | 并行度设置|否|1| +|timeout| 连接超时时间|否|10000| +|maxTotal|最大连接数|否|8| +|maxIdle|最大空闲连接数|否|8| +|minIdle|最小空闲连接数|否||0| +|masterName| 哨兵模式下的masterName|否|| ## 5.样例： From 6f8c9aa6c560c47b42bfd51684481e1054abfc20 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 21 Apr 2020 19:53:54 +0800 Subject: [PATCH 132/378] =?UTF-8?q?fix=20=E5=88=AB=E5=90=8Djoin=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/SideInfo.java | 2 +- .../flink/sql/side/rdb/async/RdbAsyncSideInfo.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index 029c86e25..4b53f77f7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -93,7 +93,7 @@ public void parseSelectFields(JoinInfo joinInfo){ if(fieldInfo.getTable().equalsIgnoreCase(sideTableName)){ String sideFieldName = sideTableInfo.getPhysicalFields().getOrDefault(fieldInfo.getFieldName(), fieldInfo.getFieldName()); fields.add(sideFieldName); - sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(sideFieldName)); + sideSelectFieldsType.put(sideTableFieldIndex, getTargetFieldType(fieldInfo.getFieldName())); sideFieldIndex.put(i, sideTableFieldIndex); sideFieldNameIndex.put(i, sideFieldName); sideTableFieldIndex++; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 9e8c13080..f87b27ae4 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -18,8 +18,6 @@ package com.dtstack.flink.sql.side.rdb.async; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; @@ -33,9 +31,11 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @@ -86,6 +86,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; + Map physicalFields = sideTableInfo.getPhysicalFields(); String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); @@ -94,7 +95,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { String rightField = right.getComponent(1).getSimple(); if (leftTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(leftField); + equalFieldList.add(physicalFields.get(leftField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -110,7 +111,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } else if (rightTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(rightField); + equalFieldList.add(physicalFields.get(rightField)); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; From ccbc0a1207e4554021d214e9e40caeb156898e92 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 22 Apr 2020 16:04:18 +0800 Subject: [PATCH 133/378] =?UTF-8?q?update=20hbaseSide=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index cc5fa319c..25af638d6 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -29,7 +29,7 @@ | columnFamily:columnName | hbase中的列族名称和列名称 | | alias | hbase 中的列对应到flink中注册的列名称 | | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| -| PRIMARY KEY(keyInfo) | 维表主键定义;hbase 维表rowkey的构造方式;可选择的构造包括 md5(alias + alias), '常量',也包括上述方式的自由组合 | +| PRIMARY KEY(keyInfo) | 维表主键定义;hbase 维表rowkey的构造方式:可选择的构造包括 md5(alias + alias), '常量',也包括上述方式的自由组合 | ## 4.参数 @@ -105,7 +105,8 @@ ROW COLUMN+CELL 2 column=wtz:info, timestamp=1587088858564, value=sql 2 column=wtz:message, timestamp=1587088840507, value=stream ``` - +在hbase中，rowKey是一个二进制码流，可以为任意字符串，flinkStreamSql读取rowKey并通过rowKey唯一确定数据，对rowKey没有任何限制,对rowKey可选择的构造包括 md5(alias + alias), '常量',也可以它们的自由组合。 +在本次案例中，rowKey为了简单，设置成了"0,1,2"这样的数值型字符，若有必要，也可以设计得更为复杂。 ### hbase异步维表关联完整案例 ``` CREATE TABLE MyTable( From 2b0af472c7d46e792c43db9070a7691fa23d1512 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 23 Apr 2020 11:02:30 +0800 Subject: [PATCH 134/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=94=99=E4=B9=B1=E9=97=AE=E9=A2=98http://re?= =?UTF-8?q?dmine.prod.dtstack.cn/issues/24996?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 971285e52..52a27fcc3 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -88,7 +88,7 @@ protected void init(SideInfo sideInfo) { @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { - CRow copyCrow = new CRow(input.row(), input.change()); + CRow copyCrow = new CRow(Row.copy(input.row()), input.change()); JsonArray inputParams = new JsonArray(); for (Integer conValIndex : sideInfo.getEqualValIndex()) { Object equalObj = copyCrow.row().getField(conValIndex); From 73ed644f80fef243dc97c5bbde86edc114e2b57a Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 23 Apr 2020 16:09:26 +0800 Subject: [PATCH 135/378] =?UTF-8?q?=E4=B9=B1=E5=BA=8F=EF=BC=8C=E5=88=AB?= =?UTF-8?q?=E5=90=8D=E5=92=8Chase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideInfo.java | 5 ++--- .../com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 2 +- .../flink/sql/side/hbase/table/HbaseSideParser.java | 1 + .../dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java | 7 +++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index d8a22e022..4b53f77f7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -132,13 +132,12 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); - Map physicalFields = sideTableInfo.getPhysicalFields(); String rightTableName = right.getComponent(0).getSimple(); String rightField = right.getComponent(1).getSimple(); if(leftTableName.equalsIgnoreCase(sideTableName)){ - equalFieldList.add(physicalFields.get(leftField)); + equalFieldList.add(leftField); int equalFieldIndex = -1; for(int i=0; i resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); + CRow inputCopy = new CRow(Row.copy(input.row()), input.change()); Map refData = Maps.newHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..3e9ab7f45 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -100,6 +100,7 @@ private void dealField(Matcher matcher, TableInfo tableInfo){ sideTableInfo.addFieldClass(fieldClass); sideTableInfo.addFieldType(fieldType); sideTableInfo.putAliasNameRef(aliasStr, fieldName); + sideTableInfo.addPhysicalMappings(aliasStr, fieldName); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index f3830793e..7aa83b3c8 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -87,7 +87,6 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; SqlIdentifier right = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[1]; - Map physicalFields = sideTableInfo.getPhysicalFields(); String leftTableName = left.getComponent(0).getSimple(); String leftField = left.getComponent(1).getSimple(); @@ -96,7 +95,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { String rightField = right.getComponent(1).getSimple(); if (leftTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(physicalFields.get(leftField)); + equalFieldList.add(leftField); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -112,7 +111,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } else if (rightTableName.equalsIgnoreCase(sideTableName)) { - equalFieldList.add(physicalFields.get(rightField)); + equalFieldList.add(rightField); int equalFieldIndex = -1; for (int i = 0; i < rowTypeInfo.getFieldNames().length; i++) { String fieldName = rowTypeInfo.getFieldNames()[i]; @@ -144,7 +143,7 @@ public String getSelectFromStatement(String tableName, List selectFields .collect(Collectors.joining(", ")); String whereClause = conditionFields.stream() - .map(f -> quoteIdentifier(f) + sqlJoinCompareOperate.get(conditionFields.indexOf(f)) + wrapperPlaceholder(f)) + .map(f -> quoteIdentifier(sideTableInfo.getPhysicalFields().getOrDefault(f, f)) + sqlJoinCompareOperate.get(conditionFields.indexOf(f)) + wrapperPlaceholder(f)) .collect(Collectors.joining(" AND ")); String predicateClause = predicateInfoes.stream() From 93844cff640be6676305ceeeddef3ea3ea655f79 Mon Sep 17 00:00:00 2001 From: maqi Date: Thu, 23 Apr 2020 16:29:57 +0800 Subject: [PATCH 136/378] pgsink sqlserversink doc --- docs/plugin/mysqlSink.md | 2 +- docs/plugin/postgresqlSink.md | 101 ++++++++++++++++++++++++++++------ docs/plugin/sqlserverSink.md | 96 ++++++++++++++++++++++++++------ 3 files changed, 164 insertions(+), 35 deletions(-) diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index 8dabea047..254f11f2a 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -36,7 +36,7 @@ CREATE TABLE tableName( |password | mysql连接密码|是|| |tableName | mysql表名称|是|| |parallelism | 并行度设置|否|1| -|batchNum | flush的大小|否|100| +|batchSize | flush的大小|否|100| |batchWaitInterval | flush的时间间隔，单位ms|否|1000| |allReplace| true:新值替换旧值|否|false| |updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| diff --git a/docs/plugin/postgresqlSink.md b/docs/plugin/postgresqlSink.md index 6d7d87834..c04ff0ec0 100644 --- a/docs/plugin/postgresqlSink.md +++ b/docs/plugin/postgresqlSink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType， + primary key (colName) )WITH( type ='postgresql', url ='jdbcUrl', @@ -25,29 +26,95 @@ CREATE TABLE tableName( | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| +| primary key | updateMode为UPSERT时，需要指定的主键信息| + ## 4.参数： |参数名称|含义|是否必填|默认值| |----|----|----|----| -| type |表明 输出表类型[postgresql]|是|| -| url | 连接postgresql数据库 jdbcUrl |是|| -| userName | postgresql连接用户名 |是|| -| password | postgresql连接密码|是|| -| tableName | postgresqll表名称|是|| -| parallelism | 并行度设置|否|1| +|type |结果表插件类型，必须为postgresql|是|| +|url | 连接postgresql数据库 jdbcUrl |是|| +|userName |postgresql连接用户名 |是|| +|password | postgresql连接密码|是|| +|tableName | postgresql表名称|是|| +|schema | postgresql表空间|否|| +|parallelism | 并行度设置|否|1| +|batchSize | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| ## 5.样例： + +回溯流删除 + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='postgresql', - url ='jdbc:postgresql://localhost:9001/test?sslmode=disable', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='postgresql', + url='jdbc:postgresql://172.16.8.193:5432/DTstack', + userName='root', + password='123456', + tableName='userInfo', + schema = 'aaa', + updateMode = 'upsert', + batchSize = '1' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + ``` + + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 删除 (1001,"maqi",null) 插入 (1001,"maqi","hz") diff --git a/docs/plugin/sqlserverSink.md b/docs/plugin/sqlserverSink.md index 8ecb14308..0ab9431e9 100644 --- a/docs/plugin/sqlserverSink.md +++ b/docs/plugin/sqlserverSink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType, + primary key (colName) )WITH( type ='sqlserver', url ='jdbcUrl', @@ -25,29 +26,90 @@ CREATE TABLE tableName( | tableName| sqlserver表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| +| primary key | updateMode为UPSERT时，需要指定的主键信息| ## 4.参数： |参数名称|含义|是否必填|默认值| |----|----|----|----| -|type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| +|type |结果表插件类型，必须为sqlserver|是|| |url | 连接sqlserver数据库 jdbcUrl |是|| -|userName | sqlserver连接用户名 |是|| -| password | sqlserver连接密码|是|| -| tableName | sqlserver表名称|是|| -| parallelism | 并行度设置|否|1| +|userName |sqlserver连接用户名 |是|| +|password | sqlserver连接密码|是|| +|tableName | sqlserver表名称|是|| +|schema | sqlserver表空间|否|| +|parallelism | 并行度设置|否|1| +|batchSize | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| + ## 5.样例： + +回溯流删除 ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='sqlserver', - url ='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='sqlserver', + url='jdbc:jtds:sqlserver://172.16.8.149:1433;DatabaseName=DTstack', + userName='sa', + password='Dtstack2018', + tableName='user', + schema = 'aaa', + updateMode = 'upsert', + batchSize = '1' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + ``` + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 删除 (1001,"maqi",null) 插入 (1001,"maqi","hz") From 83571b419945d3540c4ca3d7104318673cbb8d5a Mon Sep 17 00:00:00 2001 From: gkd Date: Thu, 23 Apr 2020 20:23:52 +0800 Subject: [PATCH 137/378] postgresql-side clickhouse-sise --- docs/plugin/clickhouseSide.md | 170 ++++++++++++++++++++++++---------- docs/plugin/postgresqlSide.md | 162 +++++++++++++++++++++++--------- 2 files changed, 239 insertions(+), 93 deletions(-) diff --git a/docs/plugin/clickhouseSide.md b/docs/plugin/clickhouseSide.md index 57e4eef6b..df050ca8b 100644 --- a/docs/plugin/clickhouseSide.md +++ b/docs/plugin/clickhouseSide.md @@ -1,6 +1,9 @@ - ## 1.格式： -``` + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + +```sql CREATE TABLE tableName( colName cloType, ... @@ -21,65 +24,132 @@ ``` # 2.支持版本 + 19.14.x、19.15.x、19.16.x - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | clickhouse表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 clickhouse |是|| - | url | 连接clickhouse数据库 jdbcUrl |是|| - | userName | clickhouse连接用户名 |是|| - | password | clickhouse连接密码|是|| - | tableName | clickhouse表名称|是|| - | tableName | clickhouse 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +clinkhose独有的参数信息 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | --------------------- | -------- | ------ | +| type | 维表类型， clinkhouse | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + +## 4.样例 + +------ + +## ALL全量维表定义 + +```sql +// 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME + )WITH( + type ='clickhouse', + url ='jdbc:clickhouse://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='all_test_clinkhousesql', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), + +### LRU异步维表定义 + +``` +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( - type='clickhouse', - url='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', + type ='clickhousesql', + url ='jdbc:clickhousesql://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='lru_test_clickhousesql', + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' + asyncPoolSize ='3', + parallelism ='1' + ); +``` + +### ClickHouseSQL异步维表关联 + +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_zftest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type ='clickhousesql', + url ='jdbc:clickhousesql://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_clickhouse_zf', + updateMode ='append', parallelism ='1', - partitionedJoin='false' + batchSize ='100', + batchWaitInterval ='1000' ); +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='clickhousesql', + url ='jdbc:clickhousesql://172.16.8.104:3306/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_clickhouse_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; ``` - diff --git a/docs/plugin/postgresqlSide.md b/docs/plugin/postgresqlSide.md index 6928042f6..b50a2de52 100644 --- a/docs/plugin/postgresqlSide.md +++ b/docs/plugin/postgresqlSide.md @@ -1,5 +1,8 @@ - ## 1.格式： + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + ``` CREATE TABLE tableName( colName cloType, @@ -21,59 +24,132 @@ ``` # 2.支持版本 + postgresql-8.2+ - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型[postgresql] |是|| - | url | 连接postgresql数据库 jdbcUrl |是|| - | userName | postgresql连接用户名 |是|| - | password | postgresql连接密码|是|| - | tableName | postgresql表名称|是|| - | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +postgresql独有的参数配置 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | ------------------ | -------- | ------ | +| type | 维表类型， mysql | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + +## 4.样例 + +### ALL全量维表定义 + +```sql + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='all_test_postgresql', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), + +### LRU异步维表定义 + +```sql +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( - type='postgresql', - url='jdbc:postgresql://localhost:9001/test?sslmode=disable', - userName='dtstack', - password='abc123', - tableName='sidetest', + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='lru_test_postgresql', + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +``` + +### PostgreSQL异步维表关联 + +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_zftest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_postgresql_zf', + updateMode ='append', parallelism ='1', - partitionedJoin='false' + batchSize ='100', + batchWaitInterval ='1000' ); +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='postgresql', + url ='jdbc:postgresql://172.16.10.194:5432/zftest', + userName ='dtstack', + password ='abc123', + tableName ='test_postgresql_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); -``` +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; +``` From e4bedb2dac3b135d331900a64ed0ef7759322e46 Mon Sep 17 00:00:00 2001 From: maqi Date: Fri, 24 Apr 2020 15:41:47 +0800 Subject: [PATCH 138/378] ck impala sink doc --- docs/plugin/clickhouseSink.md | 93 ++++++++++++++++++++++++++++++----- docs/plugin/impalaColType.md | 14 ++++++ docs/plugin/impalaSink.md | 4 +- 3 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 docs/plugin/impalaColType.md diff --git a/docs/plugin/clickhouseSink.md b/docs/plugin/clickhouseSink.md index 780ac77b0..6b4a724be 100644 --- a/docs/plugin/clickhouseSink.md +++ b/docs/plugin/clickhouseSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| clickhouse表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | clickhouse基本数据类型，不包括Array，Tuple,Nested等| ## 4.参数： @@ -36,18 +36,87 @@ CREATE TABLE tableName( | password | clickhouse 连接密码|是|| | tableName | clickhouse 表名称|是|| | parallelism | 并行度设置|否|1| +|updateMode| 只支持APPEND模式，过滤掉回撤数据||| + ## 5.样例： + + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='clickhouse', - url ='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file + id int, + name VARCHAR, + address VARCHAR +)WITH( + type='clickhouse', + url='jdbc:clickhouse://172.16.10.168:8123/tudou?charset=utf8', + userName='dtstack', + password='abc123', + tableName='MyResult', + updateMode = 'append' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + ``` + + +**Clickhouse建表语句** + + + ```aidl +CREATE TABLE tudou.MyResult (`id` Int64, `name` String, `address` String) + ENGINE = MergeTree PARTITION BY address ORDER BY id SETTINGS index_granularity = 8192 +``` + + + + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 插入 (1001,"maqi","hz") + + \ No newline at end of file diff --git a/docs/plugin/impalaColType.md b/docs/plugin/impalaColType.md new file mode 100644 index 000000000..4b84b3aa3 --- /dev/null +++ b/docs/plugin/impalaColType.md @@ -0,0 +1,14 @@ +| 支持的类型 | java对应类型 | +| ------ | ----- | +| boolean | Boolean | +| char | Character | +| double | Double| +| float | Float| +| tinyint | Byte | +| smallint | Short| +| int | Integer | +| bigint | Long | +| decimal |BigDecimal| +| string | String | +| varchar | String | +| timestamp | Timestamp | \ No newline at end of file diff --git a/docs/plugin/impalaSink.md b/docs/plugin/impalaSink.md index af3baa282..398ec6ef2 100644 --- a/docs/plugin/impalaSink.md +++ b/docs/plugin/impalaSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](docs/plugin/impalaColType.md)| ## 4.参数： @@ -45,6 +45,8 @@ CREATE TABLE tableName( | enablePartition | 是否支持分区 |否|false| | partitionFields | 分区字段名|否，enablePartition='true'时为必填|| | parallelism | 并行度设置|否|1| +| parallelism | 并行度设置|否|1| +|updateMode| 只支持APPEND模式，过滤掉回撤数据||| ## 5.样例： From f1e1d28f6caff24952e8dc747104bd5c20cce147 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 24 Apr 2020 16:02:51 +0800 Subject: [PATCH 139/378] =?UTF-8?q?where=20=E6=9D=A1=E4=BB=B6=E5=9C=A8?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E4=B8=AD=E4=B8=8B=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/SidePredicatesParser.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 4d6112aa8..7b89b25fd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -101,6 +101,8 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; + default: + break; } } @@ -128,10 +130,10 @@ private void extractPredicateInfo(SqlNode whereNode, List predica // 跳过函数 if ((((SqlBasicCall) whereNode).getOperands()[0] instanceof SqlIdentifier) - && (((SqlBasicCall) whereNode).getOperands()[1].getKind() != SqlKind.OTHER_FUNCTION)) { + && (((SqlBasicCall) whereNode).getOperands()[1].getKind() == SqlKind.LITERAL)) { fillPredicateInfoToList((SqlBasicCall) whereNode, predicatesInfoList, operatorName, operatorKind, 0, 1); } else if ((((SqlBasicCall) whereNode).getOperands()[1] instanceof SqlIdentifier) - && (((SqlBasicCall) whereNode).getOperands()[0].getKind() != SqlKind.OTHER_FUNCTION)) { + && (((SqlBasicCall) whereNode).getOperands()[0].getKind() == LITERAL)) { fillPredicateInfoToList((SqlBasicCall) whereNode, predicatesInfoList, operatorName, operatorKind, 1, 0); } } From 9a2a8915b67ca3a0b0c992420fa82a42bd6a8c41 Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 26 Apr 2020 16:52:05 +0800 Subject: [PATCH 140/378] db2 upsert --- .../dtstack/flink/sql/sink/db/DbDialect.java | 80 +++++++++++++++++++ .../com/dtstack/flink/sql/sink/db/DbSink.java | 1 + kafka09/kafka09-sink/pom.xml | 1 + kafka09/kafka09-source/pom.xml | 1 + 4 files changed, 83 insertions(+) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java index 9c7985d8f..49a3613bc 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbDialect.java @@ -19,8 +19,13 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.dtstack.flink.sql.util.DtStringUtil; +import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * Date: 2020/1/19 @@ -43,4 +48,79 @@ public String quoteIdentifier(String identifier) { return identifier; } + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + tableName = DtStringUtil.getTableFullPath(schema, tableName); + StringBuilder sb = new StringBuilder(); + sb.append("MERGE INTO " + tableName + " T1 USING " + + "(" + buildValuesStatement(fieldNames) + ") T2 (" + + buildFiledNameStatement(fieldNames) + + ") ON (" + + buildConnectionConditions(uniqueKeyFields) + ") "); + + String updateSql = buildUpdateConnection(fieldNames, uniqueKeyFields, allReplace); + + if (StringUtils.isNotEmpty(updateSql)) { + sb.append(" WHEN MATCHED THEN UPDATE SET "); + sb.append(updateSql); + } + + sb.append(" WHEN NOT MATCHED THEN " + + "INSERT (" + Arrays.stream(fieldNames).map(this::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES (" + + Arrays.stream(fieldNames).map(col -> "T2." + quoteIdentifier(col)).collect(Collectors.joining(",")) + ")"); + return Optional.of(sb.toString()); + } + + /** + * build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") + * @param fieldNames + * @param uniqueKeyFields + * @param allReplace + * @return + */ + private String buildUpdateConnection(String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + List uniqueKeyList = Arrays.asList(uniqueKeyFields); + return Arrays.stream(fieldNames) + .filter(col -> !uniqueKeyList.contains(col)) + .map(col -> buildConnectString(allReplace, col)) + .collect(Collectors.joining(",")); + } + + private String buildConnectString(boolean allReplace, String col) { + return allReplace ? quoteIdentifier("T1") + "." + quoteIdentifier(col) + " = " + quoteIdentifier("T2") + "." + quoteIdentifier(col) : + quoteIdentifier("T1") + "." + quoteIdentifier(col) + " =NVL(" + quoteIdentifier("T2") + "." + quoteIdentifier(col) + "," + + quoteIdentifier("T1") + "." + quoteIdentifier(col) + ")"; + } + + + private String buildConnectionConditions(String[] uniqueKeyFields) { + return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(",")); + } + + /** + * build sql part e.g: VALUES('1001','zs','sss') + * + * @param column destination column + * @return + */ + public String buildValuesStatement(String[] column) { + StringBuilder sb = new StringBuilder("VALUES("); + String collect = Arrays.stream(column) + .map(col -> " ? ") + .collect(Collectors.joining(", ")); + + return sb.append(collect).append(")").toString(); + } + + /** + * build sql part e.g: id, name, address + * @param column + * @return + */ + public String buildFiledNameStatement(String[] column) { + return Arrays.stream(column) + .collect(Collectors.joining(", ")); + } + + } diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 9942d4438..573a0896a 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -19,6 +19,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) + .setSchema(schema) .setTableName(tableName) .build(); diff --git a/kafka09/kafka09-sink/pom.xml b/kafka09/kafka09-sink/pom.xml index 8f52d7bc8..e01c63cc3 100644 --- a/kafka09/kafka09-sink/pom.xml +++ b/kafka09/kafka09-sink/pom.xml @@ -37,6 +37,7 @@ shade + false org.slf4j diff --git a/kafka09/kafka09-source/pom.xml b/kafka09/kafka09-source/pom.xml index deef866c9..dde2352a1 100644 --- a/kafka09/kafka09-source/pom.xml +++ b/kafka09/kafka09-source/pom.xml @@ -35,6 +35,7 @@ shade + false org.slf4j From 2e4721b6770c731015214633a181559a93bb655d Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 26 Apr 2020 17:35:58 +0800 Subject: [PATCH 141/378] db2 doc --- docs/plugin/db2Side.md | 162 ++++++++++++++++++++++++++-------- docs/plugin/db2Sink.md | 120 +++++++++++++++++++++---- docs/plugin/mysqlSink.md | 5 +- docs/plugin/postgresqlSink.md | 2 +- 4 files changed, 233 insertions(+), 56 deletions(-) diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index 8381d6206..af0c29c02 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -1,5 +1,7 @@ ## 1.格式： + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 ``` CREATE TABLE tableName( colName cloType, @@ -25,55 +27,145 @@ ## 3.表结构定义 + + [维表参数信息](docs/plugin/sideParams.md) + db2独有的参数配置： + |参数名称|含义| |----|---| - | tableName | db2表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| +| type | 维表类型， db2 |是|| +| url | 连接数据库 jdbcUrl |是|| +| userName | 连接用户名 |是|| +| password | 连接密码|是|| +| schema | 表所属scheam|否|| -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 db2 |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | db2连接用户名 |是|| - | password | db2连接密码|是|| - | tableName | db2表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 + + +## 4.样例 + +### ALL全量维表定义 ``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , PERIOD FOR SYSTEM_TIME )WITH( type='db2', - url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName='dtstack', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', password='abc123', - tableName='sidetest', + tableName='USER_INFO2', + schema = 'DTSTACK' + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='2' + ); + +``` +### LRU异步维表定义 + +``` +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='db2', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', + password='abc123', + tableName='USER_INFO2', + schema = 'DTSTACK' + partitionedJoin ='false', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='false' + asyncPoolSize ='3', + parallelism ='2' + ); + +``` + + +### DB2异步维表关联输出到Console +``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' ); + +CREATE TABLE source2( + id int, + address VARCHAR, + PERIOD FOR SYSTEM_TIME +)WITH( + type='db2', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', + password='abc123', + tableName='USER_INFO2', + schema = 'DTSTACK', + batchSize = '1' +); + + +CREATE TABLE MyResult( + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='console' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + ``` +DB2维表字段信息 + +```aidl +-- DTSTACK.USER_INFO2 definition + +CREATE TABLE "DTSTACK "."USER_INFO2" ( + "ID" INTEGER , + "NAME" VARCHAR(50 OCTETS) , + "ADDRESS" VARCHAR(50 OCTETS) ) + IN "USERSPACE1" + ORGANIZE BY ROW + ; + +GRANT CONTROL ON TABLE "DTSTACK "."USER_INFO2" TO USER "DB2INST1" +; +``` + +维表数据：(1001,maqi,hz) + +源表数据：{"name":"maqi","id":1001} + +输出结果： (1001,maqi,hz) \ No newline at end of file diff --git a/docs/plugin/db2Sink.md b/docs/plugin/db2Sink.md index 0fcbdd3d3..9544e44f1 100644 --- a/docs/plugin/db2Sink.md +++ b/docs/plugin/db2Sink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType, + [primary key (colName)] )WITH( type ='db2', url ='jdbcUrl', @@ -25,29 +26,112 @@ CREATE TABLE tableName( | tableName| db2表名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| +| primary key | updateMode为UPSERT时，需要指定的主键信息，不需要和数据库一致| + ## 4.参数： |参数名称|含义|是否必填|默认值| |----|----|----|----| -| type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| -| url | 连接db2数据库 jdbcUrl |是|| -| userName | db2连接用户名 |是|| -| password | db2连接密码|是|| -| tableName | db2表名称|是|| -| parallelism | 并行度设置|否|1| +|type |结果表插件类型，必须为db2|是|| +|url | 连接db2数据库 jdbcUrl |是|| +|userName |db2连接用户名 |是|| +|password | db2连接密码|是|| +|tableName | db2表名称|是|| +|schema | db2表空间|否|| +|parallelism | 并行度设置|否|1| +|batchSize | flush的大小|否|100| +|batchWaitInterval | flush的时间间隔，单位ms|否|1000| +|allReplace| true:新值替换旧值|否|false| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| ## 5.样例： + +回溯流删除 + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='db2', - url ='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='db2', + url='jdbc:db2://172.16.10.251:50000/mqTest', + userName='DB2INST1', + password='abc123', + tableName='USER_INFO2', + schema = 'DTSTACK', + updateMode = 'upsert', // 设置 primary key则默认为upsert + batchSize = '1' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + + ``` + + + DB2结果表建表语句： + + ```aidl +CREATE TABLE "DTSTACK "."USER_INFO2" ( + "ID" INTEGER , + "NAME" VARCHAR(50 OCTETS) , + "ADDRESS" VARCHAR(50 OCTETS) ) + IN "USERSPACE1" + ORGANIZE BY ROW + ; + +GRANT CONTROL ON TABLE "DTSTACK "."USER_INFO2" TO USER "DB2INST1" +; +``` + + + +数据结果： + +向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + +向Topic mqTest04 发送数据 {"address":"hz","id":1001} 删除 (1001,"maqi",null) 插入 (1001,"maqi","hz") \ No newline at end of file diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index 254f11f2a..ffc62792c 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -3,7 +3,8 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType + colNameX colType, + [primary key (colName)] )WITH( type ='mysql', url ='jdbcUrl', @@ -39,7 +40,7 @@ CREATE TABLE tableName( |batchSize | flush的大小|否|100| |batchWaitInterval | flush的时间间隔，单位ms|否|1000| |allReplace| true:新值替换旧值|否|false| -|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| ## 5.完整样例： ``` diff --git a/docs/plugin/postgresqlSink.md b/docs/plugin/postgresqlSink.md index c04ff0ec0..eb055cc04 100644 --- a/docs/plugin/postgresqlSink.md +++ b/docs/plugin/postgresqlSink.md @@ -43,7 +43,7 @@ CREATE TABLE tableName( |batchSize | flush的大小|否|100| |batchWaitInterval | flush的时间间隔，单位ms|否|1000| |allReplace| true:新值替换旧值|否|false| -|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|| +|updateMode| APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| ## 5.样例： From ed3aa19336fbfd1ca68d5e3f9db25cfcc7df83a2 Mon Sep 17 00:00:00 2001 From: maqi Date: Sun, 26 Apr 2020 17:57:54 +0800 Subject: [PATCH 142/378] db2 side fix --- .../com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java | 7 +++++++ .../com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java | 7 ++----- .../com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java | 8 ++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index 282ee440c..b40cfd772 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -38,4 +40,9 @@ public class Db2AllSideInfo extends RdbAllSideInfo { public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index cda2ed128..ed9fb025b 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -29,8 +29,6 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -43,8 +41,7 @@ */ public class Db2AsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(Db2AsyncReqRow.class); + private final static String DB2_PREFERRED_TEST_QUERY_SQL = "select 1 from sysibm.dual"; private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; @@ -63,7 +60,7 @@ public void open(Configuration parameters) throws Exception { .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("preferred_test_query", DB2_PREFERRED_TEST_QUERY_SQL) .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index e53dd7f27..dbb8230e7 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -38,4 +40,10 @@ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } + } From 588071175fc04ea8daa9d183de0ebfbd339ad194 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 13:53:53 +0800 Subject: [PATCH 143/378] =?UTF-8?q?=E6=9B=B4=E6=96=B0cassandra-side?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSide.md | 139 +++++++++++++++++++++++++++++------ 1 file changed, 115 insertions(+), 24 deletions(-) diff --git a/docs/plugin/cassandraSide.md b/docs/plugin/cassandraSide.md index f5210a900..5b8c955d5 100644 --- a/docs/plugin/cassandraSide.md +++ b/docs/plugin/cassandraSide.md @@ -1,5 +1,8 @@ ## 1.格式： + + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 ``` CREATE TABLE tableName( colName cloType, @@ -22,15 +25,15 @@ ``` # 2.支持版本 - cassandra-3.6.x + cassandra-3.x ## 3.表结构定义 |参数名称|含义| |----|---| - | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| + | tableName | 注册到flink的表名称(可选填;不填默认和cassandra对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| @@ -46,40 +49,128 @@ | database | cassandra表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - | maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|NONE| - | coreConnectionsPerHost | 和Cassandra集群里的每个机器都至少有2个连接|否|NONE| - | maxConnectionsPerHost | 和Cassandra集群里的每个机器都最多有6个连接|否|NONE| - | maxQueueSize | Cassandra队列大小|否|NONE| - | readTimeoutMillis | Cassandra读超时|否|NONE| - | connectTimeoutMillis | Cassandra连接超时|否|NONE| - | poolTimeoutMillis | Cassandra线程池超时|否|NONE| + | maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|1| + | coreConnectionsPerHost | Cassandra集群里的每个机器都最少连接数|否|8| + | maxConnectionsPerHost | Cassandra集群里的每个机器都最多连接数|否|32768| + | maxQueueSize | Cassandra队列大小|否|100000| + | readTimeoutMillis | Cassandra读超时|否|60000| + | connectTimeoutMillis | Cassandra连接超时|否|60000| + | poolTimeoutMillis | Cassandra线程池超时|否|60000| ---------- > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) +- NONE：不做内存缓存。每条流数据触发一次维表查询操作。 +- ALL: 任务启动时，一次性加载所有数据到内存，并进行缓存。适用于维表数据量较小的情况。 +- LRU: 任务执行时，根据维表关联条件使用异步算子加载维表数据，并进行缓存。 -## 5.样例 +## 5.维表定义样例 + +### ALL全量维表定义 ``` -create table sideTable( - CHANNEL varchar, - XCCOUNT int, - PRIMARY KEY(channel), +CREATE TABLE sideTable( + id bigint, + school varchar, + home varchar, + PRIMARY KEY(id), PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/tiezhu', + userName='dtstack', + password='abc123', + tableName='stressTest', + cache='ALL', + parallelism='1' +); +``` +### LRU异步维表定义 +``` +CREATE TABLE sideTable( + id bigint, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( type ='cassandra', - address ='172.21.32.1:9042,172.21.32.1:9042', - database ='test', - tableName ='sidetest', + address ='192.168.80.106:9042, 192.168.80.107:9042', + database ='tiezhu', + tableName ='stu', + userName='cassandra', + password='cassandra', cache ='LRU', parallelism ='1', partitionedJoin='false' - ); +); +``` +## 6.完整样例 +``` +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); +CREATE TABLE sideTable( + id bigint, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type ='cassandra', + address ='192.168.80.106:9042, 192.168.80.107:9042', + database ='tiezhu', + tableName ='stu', + userName='cassandra', + password='cassandra', + cache ='LRU', + parallelism ='1', + partitionedJoin='false' +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + message varchar + )WITH( + type ='cassandra', + address ='192.168.80.106:9042,192.168.80.107:9042', + userName='cassandra', + password='cassandra', + database ='tiezhu', + tableName ='stu_out', + parallelism ='1' +); +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from + ( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; ``` From ab67cfb197b762dc9f3740abc4dd28e546bd4875 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 13:56:25 +0800 Subject: [PATCH 144/378] =?UTF-8?q?=E6=9B=B4=E6=96=B0cassandra-side?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E4=B8=AD=E5=8F=82=E6=95=B0=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/cassandraSide.md b/docs/plugin/cassandraSide.md index 5b8c955d5..048caed62 100644 --- a/docs/plugin/cassandraSide.md +++ b/docs/plugin/cassandraSide.md @@ -49,8 +49,8 @@ | database | cassandra表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - | maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|1| - | coreConnectionsPerHost | Cassandra集群里的每个机器都最少连接数|否|8| + | maxRequestsPerConnection | 每个连接允许的并发请求数|否|1| + | coreConnectionsPerHost | 每台主机连接的核心数|否|8| | maxConnectionsPerHost | Cassandra集群里的每个机器都最多连接数|否|32768| | maxQueueSize | Cassandra队列大小|否|100000| | readTimeoutMillis | Cassandra读超时|否|60000| From cdc91af4738687d173ba715a2a6e835e7b4bd20f Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 14:03:25 +0800 Subject: [PATCH 145/378] =?UTF-8?q?=E6=9B=B4=E6=96=B0cassandra-sink?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSink.md | 125 +++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 19 deletions(-) diff --git a/docs/plugin/cassandraSink.md b/docs/plugin/cassandraSink.md index 6d6dd2301..8702acb7a 100644 --- a/docs/plugin/cassandraSink.md +++ b/docs/plugin/cassandraSink.md @@ -17,7 +17,7 @@ CREATE TABLE tableName( ``` ## 2.支持版本 - cassandra-3.6.x + cassandra-3.x ## 3.表结构定义 @@ -27,7 +27,7 @@ CREATE TABLE tableName( | colName | 列名称| | colType | 列类型 [colType支持的类型](docs/colType.md)| -## 4.参数： +## 4.参数 |参数名称|含义|是否必填|默认值| |----|----|----|----| @@ -38,26 +38,113 @@ CREATE TABLE tableName( |tableName | cassandra表名称|是|| |database | cassandra表名称|是|| |parallelism | 并行度设置|否|1| -|maxRequestsPerConnection | 每个连接最多允许64个并发请求|否|NONE| -|coreConnectionsPerHost | 和Cassandra集群里的每个机器都至少有2个连接|否|NONE| -|maxConnectionsPerHost | 和Cassandra集群里的每个机器都最多有6个连接|否|NONE| -|maxQueueSize | Cassandra队列大小|否|NONE| -|readTimeoutMillis | Cassandra读超时|否|NONE| -|connectTimeoutMillis | Cassandra连接超时|否|NONE| -|poolTimeoutMillis | Cassandra线程池超时|否|NONE| +| maxRequestsPerConnection | 每个连接允许的并发请求数|否|1| +| coreConnectionsPerHost | 每台主机连接的核心数|否|8| +| maxConnectionsPerHost | Cassandra集群里的每个机器都最多连接数|否|32768| +| maxQueueSize | Cassandra队列大小|否|100000| +| readTimeoutMillis | Cassandra读超时|否|60000| +| connectTimeoutMillis | Cassandra连接超时|否|60000| +| poolTimeoutMillis | Cassandra线程池超时|否|60000| -## 5.样例： +## 5.完整样例： ``` +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( + id bigint, + message varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type ='cassandra', + address ='192.168.80.106:9042, 192.168.80.107:9042', + database ='tiezhu', + tableName ='stu', + userName='cassandra', + password='cassandra', + cache ='LRU', + parallelism ='1', + partitionedJoin='false' +); + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR + id bigint, + name varchar, + address varchar, + message varchar )WITH( type ='cassandra', - address ='172.21.32.1:9042,172.21.32.1:9042', - userName ='dtstack', - password ='abc123', - database ='test', - tableName ='pv', + address ='192.168.80.106:9042,192.168.80.107:9042', + userName='cassandra', + password='cassandra', + database ='tiezhu', + tableName ='stu_out', parallelism ='1' - ) - ``` \ No newline at end of file +); + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from + ( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; + ``` +### 6.结果表数据展示 +``` +cqlsh:tiezhu> desc stu_out + +CREATE TABLE tiezhu.stu_out ( + id int PRIMARY KEY, + address text, + message text, + name text +) WITH bloom_filter_fp_chance = 0.01 + AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} + AND comment = '' + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} + AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND dclocal_read_repair_chance = 0.1 + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair_chance = 0.0 + AND speculative_retry = '99PERCENTILE'; + + +cqlsh:tiezhu> select * from stu_out limit 1; + + id | address | message | name +----+------------+------------------+---------- + 23 | hangzhou23 | flinkStreamSql23 | tiezhu23 + +(1 rows) +``` \ No newline at end of file From ccd5edefa8ba01b640159201dd833cf8ae78fcf2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 27 Apr 2020 14:39:06 +0800 Subject: [PATCH 146/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-sink=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/kafkaSink.md | 152 +++++++++++++++++++++++++++++++++++++++ pom.xml | 32 ++++----- 2 files changed, 168 insertions(+), 16 deletions(-) create mode 100644 docs/plugin/kafkaSink.md diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md new file mode 100644 index 000000000..8b50c9e3f --- /dev/null +++ b/docs/plugin/kafkaSink.md @@ -0,0 +1,152 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + colNameX colType, + [primary key (colName)] + )WITH( + type ='kafka09', + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + groupId='test', + parallelism ='parllNum', + timezone='Asia/Shanghai', + sourcedatatype ='json' #可不设置 + ); +``` + +## 2.支持版本 + kafka09,kafka10,kafka11及以上版本 + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName| 结果表名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](docs/colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|----|----|----| +|type |表名的输出表类型[kafka09|kafka10|kafka11|kafka]|是|| +|groupId | 需要读取的 groupId 名称|否|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|topicIsPattern | topic是否是正则表达式格式(true|false) |否| false +|offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| +|parallelism | 并行度设置|否|1| +|sourcedatatype | 数据类型|否|json| +|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' +**kafka相关参数可以自定义，使用kafka.开头即可。** +``` +kafka.consumer.id +kafka.socket.timeout.ms +kafka.fetch.message.max.bytes +kafka.num.consumer.fetchers +kafka.auto.commit.enable +kafka.auto.commit.interval.ms +kafka.queued.max.message.chunks +kafka.rebalance.max.retries +kafka.fetch.min.bytes +kafka.fetch.wait.max.ms +kafka.rebalance.backoff.ms +kafka.refresh.leader.backoff.ms +kafka.consumer.timeout.ms +kafka.exclude.internal.topics +kafka.partition.assignment.strategy +kafka.client.id +kafka.zookeeper.session.timeout.ms +kafka.zookeeper.connection.timeout.ms +kafka.zookeeper.sync.time.ms +kafka.offsets.storage +kafka.offsets.channel.backoff.ms +kafka.offsets.channel.socket.timeout.ms +kafka.offsets.commit.max.retries +kafka.dual.commit.enabled +kafka.partition.assignment.strategy +kafka.socket.receive.buffer.bytes +kafka.fetch.min.bytes +``` + +## 5.完整样例： +``` +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + groupId = 'flink_sql', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( + id bigint, + school varchar, + home varchar, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME +)WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/tiezhu', + userName='dtstack', + password='abc123', + tableName='stressTest', + cache='ALL', + parallelism='1' +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + home varchar, + school varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_out2', + parallelism = '1' +); + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.school AS school, + t2.home AS home + from( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; + ``` + +## 6.结果表数据示例： +``` +[root@node002 bin]# ./kafka-console-consumer.sh --bootstrap-server 172.16.101.224:9092 --topic tiezhu_test_out2 +{"id":122,"name":"tiezhu122","address":"hangzhou122","home":"ganzhou122","school":" ecjtu122"} +{"id":123,"name":"tiezhu123","address":"hangzhou123","home":"ganzhou123","school":" ecjtu123"} +``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2794f9004..b5b674379 100644 --- a/pom.xml +++ b/pom.xml @@ -16,24 +16,24 @@ kafka11 kafka mysql - hbase - elasticsearch5 - mongo - redis5 + + + + launcher rdb - sqlserver - oracle - cassandra - kudu - postgresql - serversocket - console - clickhouse - impala - db2 - polardb - elasticsearch6 + + + + + + + + + + + + From eb7502a01b1c5c2a8d9633e851f7008afbb0e743 Mon Sep 17 00:00:00 2001 From: gkd Date: Mon, 27 Apr 2020 16:18:02 +0800 Subject: [PATCH 147/378] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=87=E6=A1=A3db2?= =?UTF-8?q?side=20impalaside=20sqlserverside?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/db2Side.md | 141 +++++++++++++++++++++------- docs/plugin/impalaSide.md | 173 +++++++++++++++++++++++++---------- docs/plugin/sqlserverSide.md | 146 ++++++++++++++++++++++------- 3 files changed, 344 insertions(+), 116 deletions(-) diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index 8381d6206..88f0bfc0e 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -1,6 +1,9 @@ - ## 1.格式： -``` + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + +```sql CREATE TABLE tableName( colName cloType, ... @@ -20,41 +23,48 @@ ); ``` -# 2.支持版本 +## 2.支持版本 + db2 9.X - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | db2表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - + + [维表参数信息](docs/plugin/sideParams.md) + +db2独有的参数 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | ------------------ | -------- | ------ | +| type | 维表类型， db2 | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + ## 4.参数 - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 db2 |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | db2连接用户名 |是|| - | password | db2连接密码|是|| - | tableName | db2表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 +### ALL全量维表定义 + +```sql +// 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='db2', + url ='jdbc:db2://172.16.8.104:50000/test?charset=utf8', + userName ='dtstack', + password ='abc123', + tableName ='all_test_db2', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` +### LRU异步维表定义 + +```sql create table sideTable( channel varchar, xccount int, @@ -65,15 +75,78 @@ create table sideTable( url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', userName='dtstack', password='abc123', - tableName='sidetest', + tableName='lru_test_db2', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', parallelism ='1', partitionedJoin='false' ); +``` +### Db2异步维表关联 -``` +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type='db2', + url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName ='test_db2_zf', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='db2', + url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName ='test_db2_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; + +``` diff --git a/docs/plugin/impalaSide.md b/docs/plugin/impalaSide.md index 5c7479b04..500219b7f 100644 --- a/docs/plugin/impalaSide.md +++ b/docs/plugin/impalaSide.md @@ -1,6 +1,6 @@ - ## 1.格式： -``` + +```sql CREATE TABLE tableName( colName cloType, ... @@ -20,51 +20,65 @@ ); ``` -# 2.支持版本 +## 2.支持版本 + 2.10.0-cdh5.13.0 - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型[impala] |是|| - | url | 连接postgresql数据库 jdbcUrl |是|| - | userName | postgresql连接用户名 |是|| - | password | postgresql连接密码|是|| - | tableName | postgresql表名称|是|| - | authMech | 身份验证机制 (0, 1, 2, 3), 暂不支持kerberos |是|0| - | principal | kerberos用于登录的principal（authMech=1时独有） |authMech=1为必填| - | keyTabFilePath | keytab文件的路径（authMech=1时独有） |authMech=1为必填 || - | krb5FilePath | krb5.conf文件路径（authMech=1时独有） |authMech=1为必填|| - | krbServiceName | Impala服务器的Kerberos principal名称（authMech=1时独有） |authMech=1为必填|| - | krbRealm | Kerberos的域名（authMech=1时独有） |否| HADOOP.COM | - | enablePartition | 是否支持分区|否|false| - | partitionfields | 分区字段名|否,enablePartition='true'时为必填|| - | partitionFieldTypes | 分区字段类型 |否,enablePartition='true'时为必填|| - | partitionValues | 分区值|否,enablePartition='true'时为必填|| - | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +impala独有的参数配置 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| ------------------- | ------------------------------------------------------------ | --------------------------------- | ---------- | +| type | 表明维表的类型[impala] | 是 | | +| url | 连接postgresql数据库 jdbcUrl | 是 | | +| userName | postgresql连接用户名 | 是 | | +| password | postgresql连接密码 | 是 | | +| tableName | postgresql表名称 | 是 | | +| authMech | 身份验证机制 (0, 1, 2, 3), 暂不支持kerberos | 是 | 0 | +| principal | kerberos用于登录的principal（authMech=1时独有） | authMech=1为必填 | | +| keyTabFilePath | keytab文件的路径（authMech=1时独有） | authMech=1为必填 | | +| krb5FilePath | krb5.conf文件路径（authMech=1时独有） | authMech=1为必填 | | +| krbServiceName | Impala服务器的Kerberos principal名称（authMech=1时独有） | authMech=1为必填 | | +| krbRealm | Kerberos的域名（authMech=1时独有） | 否 | HADOOP.COM | +| enablePartition | 是否支持分区 | 否 | false | +| partitionfields | 分区字段名 | 否,enablePartition='true'时为必填 | | +| partitionFieldTypes | 分区字段类型 | 否,enablePartition='true'时为必填 | | +| partitionValues | 分区值 | 否,enablePartition='true'时为必填 | | +| cache | 维表缓存策略(NONE/LRU/ALL) | 否 | NONE | +| partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量) | 否 | false | + +## 4.样例 + +### ALL全量维表定义 + +```sql + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='mysql', + url ='jdbc:impala://localhost:21050/mqtest', + userName ='dtstack', + password ='1abc123', + tableName ='test_impala_all', + authMech='3', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='2', + partitionedJoin='false' + ); + ``` + +### LRU异步维表定义 + +```sql create table sideTable( channel varchar, xccount int, @@ -87,11 +101,77 @@ create table sideTable( ``` -## 6.分区样例 +### MySQL异步维表关联 -注：分区字段放在最后面，如下，name是分区字段，放在channel，xccount字段的后面 +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type='impala', + url='jdbc:impala://localhost:21050/mytest', + userName='dtstack', + password='abc123', + tableName ='test_impala_zf', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='impala', + url='jdbc:impala://localhost:21050/mytest', + userName='dtstack', + password='abc123', + tableName ='test_impala_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; ``` + +### 分区样例 + +注：分区字段放在最后面，如下，name是分区字段，放在channel，xccount字段的后面 + +```sql create table sideTable( channel varchar, xccount int, @@ -118,4 +198,3 @@ create table sideTable( ``` - diff --git a/docs/plugin/sqlserverSide.md b/docs/plugin/sqlserverSide.md index 1cab43cbd..4d50ed619 100644 --- a/docs/plugin/sqlserverSide.md +++ b/docs/plugin/sqlserverSide.md @@ -1,6 +1,9 @@ - ## 1.格式： -``` + +通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 + +```sql CREATE TABLE tableName( colName cloType, ... @@ -20,41 +23,49 @@ ); ``` -# 2.支持版本 +## 2.支持版本 + mysql-5.6.35 - + ## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | sqlserver表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 sqlserver |是|| - | url | 连接mysql数据库 jdbcUrl |是|| - | userName | sqlserver连接用户名 |是|| - | password | sqlserver连接密码|是|| - | tableName | sqlserver表名称|是|| - | tableName | sqlserver 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - -## 5.样例 + + [维表参数信息](docs/plugin/sideParams.md) + +sqlserver独有的参数配置 + +| 参数名称 | 含义 | 是否必填 | 默认值 | +| -------- | -------------------- | -------- | ------ | +| type | 维表类型， sqlserver | 是 | | +| url | 连接数据库 jdbcUrl | 是 | | +| userName | 连接用户名 | 是 | | +| password | 连接密码 | 是 | | + +## 4.样例 + +### ALL全量维表定义 + +```sql + // 定义全量维表 +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='sqlserver', + url ='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', + userName ='dtstack', + password ='abc123', + tableName ='all_test_sqlserver', + cache ='ALL', + cacheTTLMs ='60000', + parallelism ='1' + ); ``` + +### LRU异步维表定义 + +```sql create table sideTable( channel varchar, xccount int, @@ -65,7 +76,7 @@ create table sideTable( url='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', userName='dtstack', password='abc123', - tableName='sidetest', + tableName=',ru_test_sqlserver', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', @@ -76,4 +87,69 @@ create table sideTable( ``` +### PostSQL异步维表关联 + +```sql +CREATE TABLE MyTable( + id int, + name varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='cannan_yctest01', + timezone='Asia/Shanghai', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + id INT, + name VARCHAR + )WITH( + type='sqlserver', + url='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', + userName='dtstack', + password='abc123', + tableName ='test_sqlserver_zf', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +CREATE TABLE sideTable( + id INT, + name VARCHAR, + PRIMARY KEY(id) , + PERIOD FOR SYSTEM_TIME + )WITH( + type='sqlserver', + url='jdbc:jtds:sqlserver://172.16.8.104:1433;DatabaseName=mytest', + userName='dtstack', + password='abc123', + tableName ='test_sqlserver_10', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncPoolSize ='3', + parallelism ='1' + ); + +insert +into + MyResult + select + m.id, + s.name + from + MyTable m + join + sideTable s + on m.id=s.id; + +``` From 4e34155b0cdecccec1e5ff9a6fcbe88163b6e38d Mon Sep 17 00:00:00 2001 From: gkd Date: Mon, 27 Apr 2020 16:28:04 +0800 Subject: [PATCH 148/378] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/db2Side.md | 115 +---------------------------------- docs/plugin/impalaSide.md | 1 - docs/plugin/sqlserverSide.md | 1 + 3 files changed, 4 insertions(+), 113 deletions(-) diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index 4fc7541ec..af0c29c02 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -1,9 +1,5 @@ -## 1.格式： - -通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， - 维表JOIN的条件必须与`keyInfo`字段一致。 -```sql +## 1.格式： 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， 维表JOIN的条件必须与`keyInfo`字段一致。 ``` @@ -26,24 +22,10 @@ ); ``` -## 2.支持版本 - +# 2.支持版本 db2 9.X - + ## 3.表结构定义 - - [维表参数信息](docs/plugin/sideParams.md) - -db2独有的参数 - -| 参数名称 | 含义 | 是否必填 | 默认值 | -| -------- | ------------------ | -------- | ------ | -| type | 维表类型， db2 | 是 | | -| url | 连接数据库 jdbcUrl | 是 | | -| userName | 连接用户名 | 是 | | -| password | 连接密码 | 是 | | - -## 4.参数 [维表参数信息](docs/plugin/sideParams.md) @@ -58,35 +40,9 @@ db2独有的参数 | schema | 表所属scheam|否|| -### ALL全量维表定义 - -```sql -// 定义全量维表 -CREATE TABLE sideTable( - id INT, - name VARCHAR, - PRIMARY KEY(id) , - PERIOD FOR SYSTEM_TIME - )WITH( - type ='db2', - url ='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='all_test_db2', - cache ='ALL', - cacheTTLMs ='60000', - parallelism ='1' - ); -``` -### LRU异步维表定义 ## 4.样例 -```sql -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), ### ALL全量维表定义 ``` // 定义全量维表 @@ -100,7 +56,6 @@ CREATE TABLE sideTable( url='jdbc:db2://172.16.10.251:50000/mqTest', userName='DB2INST1', password='abc123', - tableName='lru_test_db2', tableName='USER_INFO2', schema = 'DTSTACK' cache ='ALL', @@ -131,7 +86,6 @@ CREATE TABLE sideTable( asyncPoolSize ='3', parallelism ='2' ); -``` ``` @@ -168,57 +122,7 @@ CREATE TABLE source2( batchSize = '1' ); -### Db2异步维表关联 - -```sql -CREATE TABLE MyTable( - id int, - name varchar - )WITH( - type ='kafka11', - bootstrapServers ='172.16.8.107:9092', - zookeeperQuorum ='172.16.8.107:2181/kafka', - offsetReset ='latest', - topic ='cannan_yctest01', - timezone='Asia/Shanghai', - enableKeyPartitions ='false', - topicIsPattern ='false', - parallelism ='1' - ); - -CREATE TABLE MyResult( - id INT, - name VARCHAR - )WITH( - type='db2', - url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName ='test_db2_zf', - updateMode ='append', - parallelism ='1', - batchSize ='100', - batchWaitInterval ='1000' - ); -CREATE TABLE sideTable( - id INT, - name VARCHAR, - PRIMARY KEY(id) , - PERIOD FOR SYSTEM_TIME - )WITH( - type='db2', - url='jdbc:db2://172.16.8.104:50000/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName ='test_db2_10', - partitionedJoin ='false', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - asyncPoolSize ='3', - parallelism ='1' - ); CREATE TABLE MyResult( id int, name VARCHAR, @@ -259,19 +163,6 @@ GRANT CONTROL ON TABLE "DTSTACK "."USER_INFO2" TO USER "DB2INST1" ; ``` -insert -into - MyResult - select - m.id, - s.name - from - MyTable m - join - sideTable s - on m.id=s.id; - -``` 维表数据：(1001,maqi,hz) 源表数据：{"name":"maqi","id":1001} diff --git a/docs/plugin/impalaSide.md b/docs/plugin/impalaSide.md index 500219b7f..dce959388 100644 --- a/docs/plugin/impalaSide.md +++ b/docs/plugin/impalaSide.md @@ -98,7 +98,6 @@ create table sideTable( partitionedJoin='false' ); - ``` ### MySQL异步维表关联 diff --git a/docs/plugin/sqlserverSide.md b/docs/plugin/sqlserverSide.md index 4d50ed619..f95d8bba5 100644 --- a/docs/plugin/sqlserverSide.md +++ b/docs/plugin/sqlserverSide.md @@ -63,6 +63,7 @@ CREATE TABLE sideTable( ); ``` + ### LRU异步维表定义 ```sql From d6f72c937960e0047ea03fcdbc07b5d7fc459aa1 Mon Sep 17 00:00:00 2001 From: maqi Date: Mon, 27 Apr 2020 16:51:49 +0800 Subject: [PATCH 149/378] mongo doc --- docs/plugin/mongoSide.md | 142 +++++++++++++++++++++++++++++---------- docs/plugin/mongoSink.md | 75 ++++++++++++++++++--- 2 files changed, 170 insertions(+), 47 deletions(-) diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 90ffa2582..92362a752 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -26,53 +26,123 @@ ## 3.表结构定义 - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| + [通用维表参数信息](docs/plugin/sideParams.md) -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type |表明 输出表类型 mongo|是|| - | address | 连接mongo数据库 jdbcUrl |是|| - | userName | mongo连接用户名|否|| - | password | mongo连接密码|否|| - | tableName | mongo表名称|是|| - | database | mongo表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - + + mongo相关参数配置： + +|参数名称|含义|是否必填|默认值| +|----|---|---|----| +| type |表明 输出表类型 mongo|是|| +| address | 连接mongo数据库 jdbcUrl |是|| +| userName | mongo连接用户名|否|| +| password | mongo连接密码|否|| +| tableName | mongo表名称|是|| +| database | mongo表名称|是|| + +## 4.样例 -## 5.样例 + +### 全量维表结构 + +```aidl +CREATE TABLE source2( + id int, + address VARCHAR, + PERIOD FOR SYSTEM_TIME +)WITH( + type ='mongo', + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo', + cache ='ALL', + parallelism ='1', + partitionedJoin='false' +); ``` -create table sideTable( - CHANNEL varchar, - XCCOUNT int, - PRIMARY KEY(channel), + +### 异步维表结构 + +```aidl +CREATE TABLE source2( + id int, + address VARCHAR, PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - database ='test', - tableName ='sidetest', + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo', cache ='LRU', parallelism ='1', partitionedJoin='false' +); + +``` + +### 异步维表关联样例 + +``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' ); +CREATE TABLE source2( + id int, + address VARCHAR, + PERIOD FOR SYSTEM_TIME +)WITH( + type ='mongo', + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo', + cache ='ALL', + parallelism ='1', + partitionedJoin='false' +); + + +CREATE TABLE MyResult( + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( + type='console' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + ``` +维表数据：{"id": 1001,"address":"hz""} + +源表数据：{"name":"maqi","id":1001} + + +输出结果： (1001,maqi,hz) + + diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index 9900dea12..8c32dc95e 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -40,17 +40,70 @@ CREATE TABLE tableName( |parallelism | 并行度设置|否|1| ## 5.样例： + +双流join并插入mongo + ``` + +CREATE TABLE source1 ( + id int, + name VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest03', + timezone='Asia/Shanghai', + topicIsPattern ='false' + ); + + + +CREATE TABLE source2( + id int, + address VARCHAR +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest04', + timezone='Asia/Shanghai', + topicIsPattern ='false' +); + + CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( + id int, + name VARCHAR, + address VARCHAR, + primary key (id) +)WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - userName ='dtstack', - password ='abc123', - database ='test', - tableName ='pv', - parallelism ='1' - ) - ``` \ No newline at end of file + address ='172.16.8.193:27017', + database ='dtstack', + tableName ='userInfo' +); + +insert into MyResult +select + s1.id, + s1.name, + s2.address +from + source1 s1 +left join + source2 s2 +on + s1.id = s2.id + + + ``` + + + 数据结果： + + 向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) + + 向Topic mqTest04 发送数据 {"address":"hz","id":1001} 插入 (1001,"maqi","hz") \ No newline at end of file From f17dc3fbf526196fb60366c0a996a7361f834ff3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 09:41:17 +0800 Subject: [PATCH 150/378] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/launcher/PluginLoadModeTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java index 0a153d018..3916d8543 100644 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java @@ -48,8 +48,22 @@ public static void testClasspathMode() throws Exception { LauncherMain.main(sql); } + + public static void testRocSql() throws Exception{ + String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/zy_sql/hbase_side.sql", "-name", "roc", + "-localSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-remoteSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-flinkconf", "/Users/roc/Documents/flink_sql/flinkconf", + "-confProp", "{\"sql.checkpoint.cleanup.mode\":\"false\",\"sql.checkpoint.interval\":10000,\"time.characteristic\":\"EventTime\"}", + "-yarnconf", "/Users/roc/Documents/flink_sql/yarnconf", + "-flinkJarPath", "/Users/roc/Documents/flink_sql/flinkJarPath", "-queue", "c", "-pluginLoadMode", "classpath"}; + System.setProperty("HADOOP_USER_NAME", "admin"); + LauncherMain.main(sql); + } + public static void main(String[] args) throws Exception { - testShipfileMode(); + testRocSql(); +// testShipfileMode(); // testClasspathMode(); } } From 9957cfd80fd4ad9d3f5e0668c717ed090327d17a Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 14:06:55 +0800 Subject: [PATCH 151/378] =?UTF-8?q?=E6=94=AF=E6=8C=81decimal=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/hbase/utils/HbaseUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java index 39df3c913..5a08b278a 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseUtils.java @@ -66,9 +66,12 @@ public static Object convertByte(byte[] hbaseData, String type){ case "double": return Bytes.toDouble(hbaseData); - + case "decimal": + return Bytes.toBigDecimal(hbaseData); + default: + throw new RuntimeException("not support type of " + type); } - - throw new RuntimeException("not support type of " + type); } + + } From dfea94ba8db66547bba0ca734880b21afcaed676 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 14:42:21 +0800 Subject: [PATCH 152/378] =?UTF-8?q?hbase=20=E7=88=B6=E7=B1=BB=E6=B2=A1?= =?UTF-8?q?=E6=9C=89open=E5=92=8C=E7=BC=93=E5=AD=98=E4=B8=AD=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=B1=BB=E5=9E=8B=E4=B8=8D=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 1 + .../sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 251b88034..0daf2bea5 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -89,6 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List Date: Tue, 28 Apr 2020 15:05:46 +0800 Subject: [PATCH 153/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9kafka=20json=20?= =?UTF-8?q?=E5=AF=B9=E7=B1=BB=E5=9E=8B=E6=AF=94=E8=BE=83=E7=9A=84=E9=83=A8?= =?UTF-8?q?=E5=88=86=E3=80=82=E4=BF=AE=E6=94=B9=20'=3D=3D'=20=E4=B8=BA=20e?= =?UTF-8?q?quals;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kafka/AbstractKafkaProducerFactory.java | 6 +- .../json/DTJsonRowSerializationSchema.java | 225 +++++++++++++++++ .../kafka/AbstractKafkaConsumerFactory.java | 6 +- .../json/DTJsonRowDeserializationSchema.java | 233 ++++++++++++++++++ 4 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 88c2ca939..9bdc0d64f 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -25,7 +25,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.AvroRowSerializationSchema; import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; +import org.apache.flink.formats.json.DTJsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.types.Row; @@ -62,9 +62,9 @@ private SerializationSchema createSerializationSchema(KafkaSinkTableInfo ka if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + serializationSchema = new DTJsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonRowSerializationSchema(typeInformation); + serializationSchema = new DTJsonRowSerializationSchema(typeInformation); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } diff --git a/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java new file mode 100644 index 000000000..f67d2353d --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.json; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.Objects; + +/** + * Serialization schema that serializes an object of Flink types into a JSON bytes. + * + *

Serializes the input Flink object into a JSON string and + * converts it into byte[]. + * + *

Result byte[] messages can be deserialized using {@link DTJsonRowDeserializationSchema}. + */ +@PublicEvolving +public class DTJsonRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = -2885556750743978636L; + + /** Type information describing the input type. */ + private final TypeInformation typeInfo; + + /** Object mapper that is used to create output JSON objects. */ + private final ObjectMapper mapper = new ObjectMapper(); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ + private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ + private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ + private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** Reusable object node. */ + private transient ObjectNode node; + + /** + * Creates a JSON serialization schema for the given type information. + * + * @param typeInfo The field names of {@link Row} are used to map to JSON properties. + */ + public DTJsonRowSerializationSchema(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + } + + /** + * Creates a JSON serialization schema for the given JSON schema. + * + * @param jsonSchema JSON schema describing the result type + * + * @see http://json-schema.org/ + */ + public DTJsonRowSerializationSchema(String jsonSchema) { + this(JsonRowSchemaConverter.convert(jsonSchema)); + } + + @Override + public byte[] serialize(Row row) { + if (node == null) { + node = mapper.createObjectNode(); + } + + try { + convertRow(node, (RowTypeInfo) typeInfo, row); + return mapper.writeValueAsBytes(node); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'. " + + "Make sure that the schema matches the input.", t); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DTJsonRowSerializationSchema that = (DTJsonRowSerializationSchema) o; + return Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo); + } + + // -------------------------------------------------------------------------------------------- + + private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { + if (reuse == null) { + reuse = mapper.createObjectNode(); + } + final String[] fieldNames = info.getFieldNames(); + final TypeInformation[] fieldTypes = info.getFieldTypes(); + + // validate the row + if (row.getArity() != fieldNames.length) { + throw new IllegalStateException(String.format( + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + } + + for (int i = 0; i < fieldNames.length; i++) { + final String name = fieldNames[i]; + + final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); + reuse.set(name, fieldConverted); + } + + return reuse; + } + + private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { + if (info.equals(Types.VOID) || object == null) { + return container.nullNode(); + } else if (info.equals(Types.BOOLEAN)) { + return container.booleanNode((Boolean) object); + } else if (info.equals(Types.STRING)) { + return container.textNode((String) object); + } else if (info.equals(Types.BIG_DEC)) { + // convert decimal if necessary + if (object instanceof BigDecimal) { + return container.numberNode((BigDecimal) object); + } + return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); + } else if (info.equals(Types.BIG_INT)) { + // convert integer if necessary + if (object instanceof BigInteger) { + return container.numberNode((BigInteger) object); + } + return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); + } else if (info.equals(Types.SQL_DATE)) { + return container.textNode(object.toString()); + } else if (info.equals(Types.SQL_TIME)) { + final Time time = (Time) object; + // strip milliseconds if possible + if (time.getTime() % 1000 > 0) { + return container.textNode(timeFormatWithMillis.format(time)); + } + return container.textNode(timeFormat.format(time)); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + return container.textNode(timestampFormat.format((Timestamp) object)); + } else if (info instanceof RowTypeInfo) { + if (reuse != null && reuse instanceof ObjectNode) { + return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); + } else { + return convertRow(null, (RowTypeInfo) info, (Row) object); + } + } else if (info instanceof ObjectArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof BasicArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType().equals(Types.BYTE)) { + return container.binaryNode((byte[]) object); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return mapper.valueToTree(object); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); + } + } + } + + private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { + if (reuse == null) { + reuse = mapper.createArrayNode(); + } else { + reuse.removeAll(); + } + + for (Object object : array) { + reuse.add(convert(reuse, null, info, object)); + } + return reuse; + } +} diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java index 2cae221cf..29cf3193b 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaConsumerFactory.java @@ -27,7 +27,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.AvroRowDeserializationSchema; import org.apache.flink.formats.csv.CsvRowDeserializationSchema; -import org.apache.flink.formats.json.JsonRowDeserializationSchema; +import org.apache.flink.formats.json.DTJsonRowDeserializationSchema; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; import org.apache.flink.types.Row; @@ -61,9 +61,9 @@ private DeserializationSchema createDeserializationSchema(KafkaSourceTableI } else if (FormatType.JSON.name().equalsIgnoreCase(kafkaSourceTableInfo.getSourceDataType())) { if (StringUtils.isNotBlank(kafkaSourceTableInfo.getSchemaString())) { - deserializationSchema = new JsonRowDeserializationSchema(kafkaSourceTableInfo.getSchemaString()); + deserializationSchema = new DTJsonRowDeserializationSchema(kafkaSourceTableInfo.getSchemaString()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - deserializationSchema = new JsonRowDeserializationSchema(typeInformation); + deserializationSchema = new DTJsonRowDeserializationSchema(typeInformation); } else { throw new IllegalArgumentException("sourceDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } diff --git a/kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java b/kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java new file mode 100644 index 000000000..6204058fe --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/org/apache/flink/formats/json/DTJsonRowDeserializationSchema.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.json; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Objects; + +/** + * Deserialization schema from JSON to Flink types. + * + *

Deserializes a byte[] message as a JSON object and reads + * the specified fields. + * + *

Failures during deserialization are forwarded as wrapped IOExceptions. + */ +@PublicEvolving +public class DTJsonRowDeserializationSchema implements DeserializationSchema { + + private static final long serialVersionUID = -228294330688809195L; + + /** Type information describing the result type. */ + private final TypeInformation typeInfo; + + /** Object mapper for parsing the JSON. */ + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** Flag indicating whether to fail on a missing field. */ + private boolean failOnMissingField; + + /** + * Creates a JSON deserialization schema for the given type information. + * + * @param typeInfo Type information describing the result type. The field names of {@link Row} + * are used to parse the JSON properties. + */ + public DTJsonRowDeserializationSchema(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + + if (!(typeInfo instanceof RowTypeInfo)) { + throw new IllegalArgumentException("Row type information expected."); + } + } + + /** + * Creates a JSON deserialization schema for the given JSON schema. + * + * @param jsonSchema JSON schema describing the result type + * + * @see http://json-schema.org/ + */ + public DTJsonRowDeserializationSchema(String jsonSchema) { + this(JsonRowSchemaConverter.convert(jsonSchema)); + } + + @Override + public Row deserialize(byte[] message) throws IOException { + try { + final JsonNode root = objectMapper.readTree(message); + return convertRow(root, (RowTypeInfo) typeInfo); + } catch (Throwable t) { + throw new IOException("Failed to deserialize JSON object.", t); + } + } + + @Override + public boolean isEndOfStream(Row nextElement) { + return false; + } + + @Override + public TypeInformation getProducedType() { + return typeInfo; + } + + /** + * Configures the failure behaviour if a JSON field is missing. + * + *

By default, a missing field is ignored and the field is set to null. + * + * @param failOnMissingField Flag indicating whether to fail or not on a missing field. + */ + public void setFailOnMissingField(boolean failOnMissingField) { + this.failOnMissingField = failOnMissingField; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DTJsonRowDeserializationSchema that = (DTJsonRowDeserializationSchema) o; + return failOnMissingField == that.failOnMissingField && Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo, failOnMissingField); + } + + // -------------------------------------------------------------------------------------------- + + private Object convert(JsonNode node, TypeInformation info) { + if (info.equals(Types.VOID) || node.isNull()) { + return null; + } else if (info.equals(Types.BOOLEAN)) { + return node.asBoolean(); + } else if (info.equals(Types.STRING)) { + return node.asText(); + } else if (info.equals(Types.BIG_DEC)) { + return node.decimalValue(); + } else if (info.equals(Types.BIG_INT)) { + return node.bigIntegerValue(); + } else if (info.equals(Types.SQL_DATE)) { + return Date.valueOf(node.asText()); + } else if (info.equals(Types.SQL_TIME)) { + // according to RFC 3339 every full-time must have a timezone; + // until we have full timezone support, we only support UTC; + // users can parse their time as string as a workaround + final String time = node.asText(); + if (time.indexOf('Z') < 0 || time.indexOf('.') >= 0) { + throw new IllegalStateException( + "Invalid time format. Only a time in UTC timezone without milliseconds is supported yet. " + + "Format: HH:mm:ss'Z'"); + } + return Time.valueOf(time.substring(0, time.length() - 1)); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + // according to RFC 3339 every date-time must have a timezone; + // until we have full timezone support, we only support UTC; + // users can parse their time as string as a workaround + final String timestamp = node.asText(); + if (timestamp.indexOf('Z') < 0) { + throw new IllegalStateException( + "Invalid timestamp format. Only a timestamp in UTC timezone is supported yet. " + + "Format: yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + } + return Timestamp.valueOf(timestamp.substring(0, timestamp.length() - 1).replace('T', ' ')); + } else if (info instanceof RowTypeInfo) { + return convertRow(node, (RowTypeInfo) info); + } else if (info instanceof ObjectArrayTypeInfo) { + return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof BasicArrayTypeInfo) { + return convertObjectArray(node, ((BasicArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof PrimitiveArrayTypeInfo && + ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return convertByteArray(node); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return objectMapper.treeToValue(node, info.getTypeClass()); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for node: " + node); + } + } + } + + private Row convertRow(JsonNode node, RowTypeInfo info) { + final String[] names = info.getFieldNames(); + final TypeInformation[] types = info.getFieldTypes(); + + final Row row = new Row(names.length); + for (int i = 0; i < names.length; i++) { + final String name = names[i]; + final JsonNode subNode = node.get(name); + if (subNode == null) { + if (failOnMissingField) { + throw new IllegalStateException( + "Could not find field with name '" + name + "'."); + } else { + row.setField(i, null); + } + } else { + row.setField(i, convert(subNode, types[i])); + } + } + + return row; + } + + private Object convertObjectArray(JsonNode node, TypeInformation elementType) { + final Object[] array = (Object[]) Array.newInstance(elementType.getTypeClass(), node.size()); + for (int i = 0; i < node.size(); i++) { + array[i] = convert(node.get(i), elementType); + } + return array; + } + + private Object convertByteArray(JsonNode node) { + try { + return node.binaryValue(); + } catch (IOException e) { + throw new RuntimeException("Unable to deserialize byte array.", e); + } + } +} From ee766b80005bb4dee464fb67d8d48e6ef39deeaa Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 28 Apr 2020 20:25:29 +0800 Subject: [PATCH 154/378] =?UTF-8?q?fix=20=E7=BC=93=E5=AD=98=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=B2=A1=E6=9C=89=E8=A7=A3=E6=9E=90=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 0daf2bea5..47adc88b9 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -148,7 +148,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce return; } else if (ECacheContentType.SingleLine == val.getType()) { try { - Row row = fillData(inputCopy.row(), val); + Row row = fillData(inputCopy.row(), val.getContent()); resultFuture.complete(Collections.singleton(new CRow(row, inputCopy.change()))); } catch (Exception e) { dealFillDataError(resultFuture, e, inputCopy); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..901c000f5 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -61,6 +61,7 @@ public HbaseSideParser() { public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); + parseCacheProp(hbaseTableInfo, props); parseFieldsInfo(fieldsInfo, hbaseTableInfo); hbaseTableInfo.setTableName((String) props.get(TABLE_NAME_KEY.toLowerCase())); hbaseTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PARALLELISM_KEY.toLowerCase()))); From bbb4e9c6b8652f7c8e34412df6b12df002f8ead5 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 29 Apr 2020 15:26:39 +0800 Subject: [PATCH 155/378] =?UTF-8?q?update=20kudu-side=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/kuduSide.md | 151 +++++++++++++++++++++++++--------------- pom.xml | 4 +- 2 files changed, 98 insertions(+), 57 deletions(-) diff --git a/docs/plugin/kuduSide.md b/docs/plugin/kuduSide.md index 718d43117..a45300deb 100644 --- a/docs/plugin/kuduSide.md +++ b/docs/plugin/kuduSide.md @@ -1,53 +1,34 @@ - -## 1.格式： -All: -``` -create table sideTable( - id int, - tablename1 VARCHAR, - PRIMARY KEY(id), - PERIOD FOR SYSTEM_TIME - )WITH( - type='kudu', - kuduMasters ='ip1,ip2,ip3', - tableName ='impala::default.testSide', - cache ='ALL', - primaryKey='id，xx', - lowerBoundPrimaryKey='10,xx', - upperBoundPrimaryKey='15,xx', - workerCount='1', - defaultOperationTimeoutMs='600000', - defaultSocketReadTimeoutMs='6000000', - batchSizeBytes='100000000', - limitNum='1000', - isFaultTolerant='false', - partitionedJoin='false' - ); -``` -LRU: +## 1.格式 + 通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段， + 维表JOIN的条件必须与`keyInfo`字段一致。 ``` -create table sideTable( - id int, - tablename1 VARCHAR, - PRIMARY KEY(id), + CREATE TABLE tableName( + colName cloType, + ... + PRIMARY KEY(colName1,colName2) , PERIOD FOR SYSTEM_TIME - )WITH( - type='kudu', + )WITH( + type ='kudu', kuduMasters ='ip1,ip2,ip3', tableName ='impala::default.testSide', - cache ='LRU', + primaryKey='id，xx', + lowerBoundPrimaryKey='10,xx', + upperBoundPrimaryKey='15,xx', workerCount='1', defaultOperationTimeoutMs='600000', defaultSocketReadTimeoutMs='6000000', batchSizeBytes='100000000', limitNum='1000', isFaultTolerant='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', partitionedJoin='false' - ); - ``` - + ); +``` ## 2.支持版本 -kudu 1.10.0+cdh6.2.0 + kudu 1.10.0+cdh6.2.0 ## 3.表结构定义 @@ -55,12 +36,13 @@ kudu 1.10.0+cdh6.2.0 |----|---| | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型 [colType支持的类型](colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| -## 3.参数 +## 4.参数 +参数详细说明请看[参数详细说明](./sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|-----| @@ -78,17 +60,25 @@ kudu 1.10.0+cdh6.2.0 | isFaultTolerant |查询是否容错 查询失败是否扫描第二个副本 默认false 容错 | 否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - -------------- -> 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) -## 4.样例 -All: +## 5.样例 +### LRU维表示例 +``` +create table sideTable( + id int, + tablename1 VARCHAR, + PRIMARY KEY(id), + PERIOD FOR SYSTEM_TIME + )WITH( + type='kudu', + kuduMasters ='ip1,ip2,ip3', + tableName ='impala::default.testSide', + cache ='LRU', + partitionedJoin='false' + ); +``` +### ALL维表示例 ``` create table sideTable( id int, @@ -106,19 +96,70 @@ create table sideTable( partitionedJoin='false' ); ``` -LRU: + +## 6.kudu异步关联完整样例 + ``` -create table sideTable( +CREATE TABLE MyTable( + id bigint, + name varchar, + address varchar +)WITH( + type = 'kafka10', + bootstrapServers = '172.16.101.224:9092', + zookeeperQuorm = '172.16.100.188:2181/kafka', + offsetReset = 'latest', + topic = 'tiezhu_test_in2', + timezone = 'Asia/Shanghai', + topicIsPattern = 'false', + parallelism = '1' +); + +CREATE TABLE sideTable( id int, - tablename1 VARCHAR, + message varchar, PRIMARY KEY(id), PERIOD FOR SYSTEM_TIME - )WITH( +)WITH( type='kudu', kuduMasters ='ip1,ip2,ip3', tableName ='impala::default.testSide', cache ='LRU', partitionedJoin='false' - ); - ``` +); + +CREATE TABLE MyResult( + id bigint, + name varchar, + address varchar, + message varchar +)WITH( + type ='console', + address ='192.168.80.106:9042,192.168.80.107:9042', + userName='cassandra', + password='cassandra', + database ='tiezhu', + tableName ='stu_out', + parallelism ='1' +); + +insert +into + MyResult + select + t1.id AS id, + t1.name AS name, + t1.address AS address, + t2.message AS message + from( + select + id, + name, + address + from + MyTable + ) t1 + join sideTable t2 + on t1.id = t2.id; +``` diff --git a/pom.xml b/pom.xml index b5b674379..2b2d0a986 100644 --- a/pom.xml +++ b/pom.xml @@ -25,10 +25,10 @@ - + kudu - + console From 1a361e1e3c86a0e09b3460ab05b43540f002cd01 Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 30 Apr 2020 19:19:43 +0800 Subject: [PATCH 156/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=A1=A8=E5=90=8D?= =?UTF-8?q?=E7=A7=B0=E5=92=8C=20schema=20=E6=B7=BB=E5=8A=A0=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/util/DtStringUtil.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..f84cc9ae4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -261,14 +261,43 @@ public static String firstUpperCase(String str) { } public static String getTableFullPath(String schema, String tableName) { + String[] tableInfoSplit = StringUtils.split(tableName, "."); + //表明表信息带了schema + if(tableInfoSplit.length == 2){ + schema = tableInfoSplit[0]; + tableName = tableInfoSplit[1]; + } + + //清理首个字符" 和最后字符 " + schema = rmStrQuote(schema); + tableName = rmStrQuote(tableName); + if (StringUtils.isEmpty(schema)){ return addQuoteForStr(tableName); } + String schemaAndTabName = addQuoteForStr(schema) + "." + addQuoteForStr(tableName); return schemaAndTabName; } + /** + * 清理首个字符" 和最后字符 " + */ + public static String rmStrQuote(String str){ + if(StringUtils.isEmpty(str)){ + return str; + } + + if(str.startsWith("\"")){ + str = str.substring(1); + } + + if(str.endsWith("\"")){ + str = str.substring(0, str.length()-1); + } + return str; + } public static String addQuoteForStr(String column) { return getStartQuote() + column + getEndQuote(); From cc228def0670a8b6c62add4e4be267c437f3e4ed Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 6 May 2020 19:13:48 +0800 Subject: [PATCH 157/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=AD=E5=8E=9Fhba?= =?UTF-8?q?se=E7=BB=B4=E8=A1=A8=E5=8F=96=E5=87=BA=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=9C=89=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98=E5=90=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 664fe6abd..9b79c5c3d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -33,6 +33,7 @@ import com.dtstack.flink.sql.factory.DTThreadFactory; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -83,7 +84,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List Date: Mon, 11 May 2020 15:06:00 +0800 Subject: [PATCH 158/378] =?UTF-8?q?redis=20=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/launcher/PluginLoadModeTest.java | 15 +++++++++- pom.xml | 28 +++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java index 0a153d018..923c74116 100644 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java @@ -48,8 +48,21 @@ public static void testClasspathMode() throws Exception { LauncherMain.main(sql); } + public static void testroc() throws Exception { + String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/result/redis_result.sql", "-name", "roc", + "-localSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-remoteSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", + "-flinkconf", "//Users/roc/Documents/flink_sql/flinkconf", + "-confProp", "{\"sql.checkpoint.cleanup.mode\":\"false\",\"sql.checkpoint.interval\":10000,\"time.characteristic\":\"EventTime\"}", + "-yarnconf", "/Users/roc/Documents/flink_sql/yarnconf", + "-flinkJarPath", "/Users/roc/Documents/flink_sql/flinkJarPath", "-queue", "c", "-pluginLoadMode", "classpath"}; + System.setProperty("HADOOP_USER_NAME", "admin"); + LauncherMain.main(sql); + } + public static void main(String[] args) throws Exception { - testShipfileMode(); + testroc(); +// testShipfileMode(); // testClasspathMode(); } } diff --git a/pom.xml b/pom.xml index 2b2d0a986..2794f9004 100644 --- a/pom.xml +++ b/pom.xml @@ -16,24 +16,24 @@ kafka11 kafka mysql - - - - + hbase + elasticsearch5 + mongo + redis5 launcher rdb - - - + sqlserver + oracle + cassandra kudu - - + postgresql + serversocket console - - - - - + clickhouse + impala + db2 + polardb + elasticsearch6 From 966b17136fb06ad52394908c0f20dd440ebe7911 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 11 May 2020 15:06:17 +0800 Subject: [PATCH 159/378] =?UTF-8?q?=E8=A1=A5=E5=85=85redis=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E8=A1=A8=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSink.md | 73 ++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index b43cb7377..224c26b2f 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -13,8 +13,6 @@ CREATE TABLE tableName( tableName ='tableName', parallelism ='parllNum' ); - - ``` ## 2.支持版本 @@ -50,17 +48,64 @@ redis5.0 ## 5.样例： ``` + CREATE TABLE MyTable( + name varchar, + channel varchar + )WITH( + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - channel varchar, - pv varchar, - PRIMARY KEY(channel) - )WITH( - type='redis', - url='172.16.10.79:6379', - password='abc123', - database='0', - redisType='1', - tableName='sinktoredis' - ); + channel VARCHAR, + pv VARCHAR + )WITH( + type ='redis', + redisType ='1', + url ='172.16.8.109:6379', + tableName ='resultTable', + partitionedJoin ='false', + parallelism ='1', + database ='0', + timeout ='10000', + maxTotal ='60000', + maxIdle='8', + minIdle='0' + ); + + insert + into + MyResult + select + channel, + name as pv + from + MyTable a + ``` + +## 6.redis完整样例 +### redis数据说明 +redis使用k-v格式存储，key的构建格式为tableName:privateKey:privateKeyValue:columnName, value=columnValue - ``` \ No newline at end of file +### 源表数据内容 +``` +{"name":"roc","channel":"daishu","age":2} +``` +### redis实际数据内容 +``` +127.0.0.1:6379> keys * +1) "resultTable:name:roc:name" +2) "resultTable:name:roc:channel" +127.0.0.1:6379> get "resultTable:name:roc:name" +"roc" +127.0.0.1:6379> get "resultTable:name:roc:channel" +"daishu" +``` \ No newline at end of file From 85aae4721ccbb27a0319fd8c1d40327a67545ef5 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 15:34:35 +0800 Subject: [PATCH 160/378] =?UTF-8?q?=E5=9B=9E=E6=BB=9A=205f30a74b=20?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=20'=E5=9B=9E=E6=BB=9A=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84test=E5=88=86=E6=94=AF=E5=8F=98=E6=9B=B4?= =?UTF-8?q?=E5=86=85=E5=AE=B9'=E6=97=A0=E6=B3=95=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 10 + .../side/cassandra/CassandraAllReqRow.java | 10 +- .../side/cassandra/CassandraAllSideInfo.java | 10 +- .../side/cassandra/CassandraAsyncReqRow.java | 11 +- .../cassandra/CassandraAsyncSideInfo.java | 18 +- .../cassandra/table/CassandraSideParser.java | 17 +- .../table/CassandraSideTableInfo.java | 4 +- .../sink/cassandra/CassandraOutputFormat.java | 4 +- .../sql/sink/cassandra/CassandraSink.java | 4 +- .../cassandra/table/CassandraSinkParser.java | 10 +- .../cassandra/table/CassandraTableInfo.java | 4 +- ci/sonar_notify.sh | 14 + .../side/clickhouse/ClickhouseAllReqRow.java | 17 +- .../clickhouse/ClickhouseAllSideInfo.java | 4 +- .../clickhouse/ClickhouseAsyncReqRow.java | 6 +- .../clickhouse/ClickhouseAsyncSideInfo.java | 4 +- .../table/ClickhouseSideParser.java | 6 +- .../sql/sink/clickhouse/ClickhouseSink.java | 9 +- .../table/ClickhouseSinkParser.java | 6 +- .../sql/sink/console/ConsoleOutputFormat.java | 6 +- .../flink/sql/sink/console/ConsoleSink.java | 4 +- .../sink/console/table/ConsoleSinkParser.java | 10 +- .../sink/console/table/ConsoleTableInfo.java | 4 +- .../sink/console/table/TablePrintUtil.java | 23 +- core/pom.xml | 10 +- .../main/java/com/dtstack/flink/sql/Main.java | 5 +- .../sql/classloader/ClassLoaderManager.java | 4 +- .../flink/sql/constrant/ConfigConstrant.java | 13 +- .../dtstack/flink/sql/enums/ColumnType.java | 63 ++- .../flink/sql/enums/ECacheContentType.java | 9 + .../dtstack/flink/sql/enums/ECacheType.java | 13 +- .../flink/sql/enums/EDatabaseType.java | 9 + .../flink/sql/enums/EPluginLoadMode.java | 6 + .../flink/sql/enums/EStateBackend.java | 9 + .../environment/StreamEnvConfigManager.java | 59 ++- .../flink/sql/exec/ExecuteProcessHelper.java | 76 ++-- .../dtstack/flink/sql/exec/ParamsInfo.java | 1 - .../format/SerializationMetricWrapper.java | 11 +- .../DtNestRowDeserializationSchema.java | 11 +- .../flink/sql/option/OptionParser.java | 21 +- ...t.java => AbstractDtRichOutputFormat.java} | 2 +- .../flink/sql/parser/CreateFuncParser.java | 8 +- .../dtstack/flink/sql/parser/SqlParser.java | 12 +- .../com/dtstack/flink/sql/parser/SqlTree.java | 8 +- ...leInfo.java => AbstractSideTableInfo.java} | 4 +- .../{AllReqRow.java => BaseAllReqRow.java} | 18 +- ...{AsyncReqRow.java => BaseAsyncReqRow.java} | 18 +- .../side/{SideInfo.java => BaseSideInfo.java} | 22 +- .../flink/sql/side/JoinNodeDealer.java | 10 +- .../flink/sql/side/ParserJoinField.java | 2 + .../flink/sql/side/SidePredicatesParser.java | 8 +- .../dtstack/flink/sql/side/SideSQLParser.java | 7 +- .../dtstack/flink/sql/side/SideSqlExec.java | 41 +- .../flink/sql/side/StreamSideFactory.java | 10 +- ...sSideCache.java => AbstractSideCache.java} | 8 +- .../flink/sql/side/cache/LRUSideCache.java | 6 +- .../sql/side/operator/SideAsyncOperator.java | 16 +- .../operator/SideWithAllCacheOperator.java | 18 +- .../flink/sql/sink/IStreamSinkGener.java | 4 +- .../flink/sql/sink/StreamSinkFactory.java | 12 +- .../flink/sql/source/IStreamSourceGener.java | 4 +- .../flink/sql/source/StreamSourceFactory.java | 12 +- ...rser.java => AbstractSideTableParser.java} | 49 +-- ...eParser.java => AbstractSourceParser.java} | 16 +- ...Info.java => AbstractSourceTableInfo.java} | 2 +- ...{TableInfo.java => AbstractTableInfo.java} | 2 +- ...rser.java => AbstractTableInfoParser.java} | 20 +- ...leParser.java => AbstractTableParser.java} | 20 +- ...Info.java => AbstractTargetTableInfo.java} | 2 +- .../sql/table/ITableFieldDealHandler.java | 2 +- .../com/dtstack/flink/sql/util/ClassUtil.java | 3 +- .../com/dtstack/flink/sql/util/DateUtil.java | 408 ++++++++---------- .../dtstack/flink/sql/util/DtStringUtil.java | 8 +- .../com/dtstack/flink/sql/util/JDBCUtils.java | 4 +- .../com/dtstack/flink/sql/util/MathUtil.java | 2 +- .../dtstack/flink/sql/util/PluginUtil.java | 10 +- ....java => AbstractCustomerWaterMarker.java} | 4 +- .../CustomerWaterMarkerForLong.java | 2 +- .../CustomerWaterMarkerForTimeStamp.java | 3 +- .../sql/watermarker/WaterMarkerAssigner.java | 8 +- .../sql/side/SidePredicatesParserTest.java | 4 +- .../flink/sql/side/db2/Db2AllReqRow.java | 15 +- .../flink/sql/side/db2/Db2AllSideInfo.java | 4 +- .../flink/sql/side/db2/Db2AsyncReqRow.java | 6 +- .../flink/sql/side/db2/Db2AsyncSideInfo.java | 4 +- .../sql/side/db2/table/Db2SideParser.java | 6 +- .../com/dtstack/flink/sql/sink/db/DbSink.java | 9 +- .../flink/sql/sink/db/table/DbSinkParser.java | 6 +- docs/kafkaSink.md | 223 ++++++++++ docs/plugin/kafkaSource.md | 226 ++-------- docs/plugin/mongoSide.md | 5 +- docs/plugin/mongoSink.md | 7 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../Elasticsearch6AllReqRow.java | 21 +- .../Elasticsearch6AllSideInfo.java | 11 +- .../Elasticsearch6AsyncReqRow.java | 11 +- .../Elasticsearch6AsyncSideInfo.java | 13 +- .../table/Elasticsearch6SideParser.java | 8 +- .../table/Elasticsearch6SideTableInfo.java | 4 +- .../sql/side/elasticsearch6/util/Es6Util.java | 10 +- .../sink/elasticsearch/ElasticsearchSink.java | 4 +- .../table/ElasticsearchSinkParser.java | 8 +- .../table/ElasticsearchTableInfo.java | 4 +- .../flink/sql/side/hbase/HbaseAllReqRow.java | 22 +- .../sql/side/hbase/HbaseAllSideInfo.java | 10 +- .../sql/side/hbase/HbaseAsyncReqRow.java | 14 +- .../sql/side/hbase/HbaseAsyncSideInfo.java | 10 +- ...ler.java => AbstractRowKeyModeDealer.java} | 12 +- .../PreRowKeyModeDealerDealer.java | 12 +- .../rowkeydealer/RowKeyEqualModeDealer.java | 9 +- ...ator.java => AbstractReplaceOperator.java} | 4 +- .../sql/side/hbase/Md5ReplaceOperator.java | 2 +- .../sql/side/hbase/enums/EReplaceOpType.java | 9 +- .../sql/side/hbase/enums/EReplaceType.java | 9 + .../sql/side/hbase/table/HbaseSideParser.java | 12 +- .../side/hbase/table/HbaseSideTableInfo.java | 4 +- .../sql/side/hbase/utils/HbaseUtils.java | 2 + .../sql/sink/hbase/HbaseOutputFormat.java | 11 +- .../flink/sql/sink/hbase/HbaseSink.java | 5 +- .../sql/sink/hbase/table/HbaseSinkParser.java | 10 +- .../sql/sink/hbase/table/HbaseTableInfo.java | 4 +- .../sql/side/impala/ImpalaAllReqRow.java | 13 +- .../sql/side/impala/ImpalaAllSideInfo.java | 7 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 12 +- .../sql/side/impala/ImpalaAsyncSideInfo.java | 6 +- .../side/impala/table/ImpalaSideParser.java | 6 +- .../flink/sql/sink/impala/ImpalaSink.java | 14 +- .../sink/impala/table/ImpalaSinkParser.java | 7 +- .../kafka/AbstractKafkaProducerFactory.java | 37 +- .../sql/sink/kafka/AbstractKafkaSink.java | 148 +++++++ .../sink/kafka/CustomerFlinkPartition.java | 8 +- .../CustomerKeyedSerializationSchema.java | 35 +- .../AvroCRowSerializationSchema.java | 366 ++++++++++++++++ .../CsvCRowSerializationSchema.java | 374 ++++++++++++++++ .../JsonCRowSerializationSchema.java | 234 ++++++++++ .../sql/sink/kafka/table/KafkaSinkParser.java | 13 +- .../sink/kafka/table/KafkaSinkTableInfo.java | 44 +- .../sql/source/kafka/AbstractKafkaSource.java | 117 +++++ .../sql/source/kafka/enums/EKafkaOffset.java | 43 +- .../source/kafka/table/KafkaSourceParser.java | 42 +- .../kafka/table/KafkaSourceTableInfo.java | 181 ++++---- .../flink/sql/sink/kafka/KafkaProducer.java | 6 +- .../sql/sink/kafka/KafkaProducerFactory.java | 4 +- .../flink/sql/sink/kafka/KafkaSink.java | 118 +---- .../flink/sql/source/kafka/KafkaSource.java | 78 +--- .../flink/sql/sink/kafka/KafkaProducer09.java | 5 +- .../sink/kafka/KafkaProducer09Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 124 +----- .../flink/sql/source/kafka/KafkaSource.java | 76 +--- .../sql/sink/kafka/KafkaProducer010.java | 6 +- .../sink/kafka/KafkaProducer010Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 122 +----- .../flink/sql/source/kafka/KafkaSource.java | 89 +--- .../sql/sink/kafka/KafkaProducer011.java | 6 +- .../sink/kafka/KafkaProducer011Factory.java | 3 +- .../flink/sql/sink/kafka/KafkaSink.java | 121 +----- .../source/kafka/KafkaConsumer011Factory.java | 2 +- .../flink/sql/source/kafka/KafkaSource.java | 92 +--- .../flink/sql/side/kudu/KuduAllReqRow.java | 24 +- .../flink/sql/side/kudu/KuduAllSideInfo.java | 10 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 26 +- .../sql/side/kudu/KuduAsyncSideInfo.java | 10 +- .../sql/side/kudu/table/KuduSideParser.java | 11 +- .../side/kudu/table/KuduSideTableInfo.java | 4 +- .../flink/sql/side/kudu/utils/KuduUtil.java | 1 + .../flink/sql/sink/kudu/KuduOutputFormat.java | 8 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 4 +- .../sql/sink/kudu/table/KuduSinkParser.java | 11 +- .../sql/sink/kudu/table/KuduTableInfo.java | 4 +- .../sql/launcher/ClusterClientFactory.java | 8 +- .../perjob/PerJobClusterClientBuilder.java | 3 +- .../sql/launcher/perjob/PerJobSubmitter.java | 1 - .../flink/sql/side/mongo/MongoAllReqRow.java | 62 +-- .../sql/side/mongo/MongoAllSideInfo.java | 10 +- .../sql/side/mongo/MongoAsyncReqRow.java | 83 ++-- .../sql/side/mongo/MongoAsyncSideInfo.java | 13 +- .../sql/side/mongo/table/MongoSideParser.java | 12 +- .../side/mongo/table/MongoSideTableInfo.java | 4 +- .../flink/sql/side/mongo/utils/MongoUtil.java | 1 + .../sql/sink/mongo/MongoOutputFormat.java | 44 +- .../flink/sql/sink/mongo/MongoSink.java | 4 +- .../sql/sink/mongo/table/MongoSinkParser.java | 10 +- .../sql/sink/mongo/table/MongoTableInfo.java | 4 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 --- .../flink/sql/side/mysql/MysqlAllReqRow.java | 12 +- .../sql/side/mysql/MysqlAllSideInfo.java | 4 +- .../sql/side/mysql/MysqlAsyncReqRow.java | 11 +- .../sql/side/mysql/MysqlAsyncSideInfo.java | 4 +- .../sql/side/mysql/table/MysqlSideParser.java | 6 +- .../flink/sql/sink/mysql/MysqlSink.java | 6 +- .../sql/sink/mysql/table/MysqlSinkParser.java | 6 +- .../test/java/com/dtstack/flinkx/AppTest.java | 58 --- .../sql/side/oracle/OracleAllReqRow.java | 12 +- .../sql/side/oracle/OracleAllSideInfo.java | 5 +- .../sql/side/oracle/OracleAsyncReqRow.java | 11 +- .../sql/side/oracle/OracleAsyncSideInfo.java | 9 +- .../side/oracle/table/OracleSideParser.java | 6 +- .../flink/sql/sink/oracle/OracleDialect.java | 8 +- .../flink/sql/sink/oracle/OracleSink.java | 6 +- .../sink/oracle/table/OracleSinkParser.java | 6 +- .../sql/side/polardb/PolardbAllReqRow.java | 34 +- .../sql/side/polardb/PolardbAllSideInfo.java | 26 +- .../sql/side/polardb/PolardbAsyncReqRow.java | 28 +- .../side/polardb/PolardbAsyncSideInfo.java | 26 +- .../side/polardb/table/PolardbSideParser.java | 29 +- .../flink/sql/sink/polardb/PolardbSink.java | 30 +- .../sink/polardb/table/PolardbSinkParser.java | 28 +- pom.xml | 16 +- .../side/postgresql/PostgresqlAllReqRow.java | 12 +- .../postgresql/PostgresqlAllSideInfo.java | 4 +- .../postgresql/PostgresqlAsyncReqRow.java | 6 +- .../postgresql/PostgresqlAsyncSideInfo.java | 4 +- .../table/PostgresqlSideParser.java | 6 +- .../sql/sink/postgresql/PostgresqlSink.java | 6 +- .../table/PostgresqlSinkParser.java | 6 +- ...lReqRow.java => AbstractRdbAllReqRow.java} | 12 +- .../sql/side/rdb/all/RdbAllSideInfo.java | 15 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 28 +- .../sql/side/rdb/async/RdbAsyncSideInfo.java | 16 +- .../provider/DTC3P0DataSourceProvider.java | 10 +- .../sql/side/rdb/table/RdbSideParser.java | 8 +- .../sql/side/rdb/table/RdbSideTableInfo.java | 4 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 1 + .../{RdbSink.java => AbstractRdbSink.java} | 19 +- .../flink/sql/sink/rdb/JDBCOptions.java | 22 +- .../rdb/format/AbstractJDBCOutputFormat.java | 14 +- .../rdb/format/JDBCUpsertOutputFormat.java | 14 +- .../rdb/format/RetractJDBCOutputFormat.java | 0 .../sql/sink/rdb/table/RdbSinkParser.java | 9 +- .../sql/sink/rdb/table/RdbTableInfo.java | 6 +- ...tWriter.java => AbstractUpsertWriter.java} | 86 ++-- .../sql/sink/rdb/writer/AppendOnlyWriter.java | 13 +- .../flink/sql/side/redis/RedisAllReqRow.java | 32 +- .../sql/side/redis/RedisAllSideInfo.java | 14 +- .../sql/side/redis/RedisAsyncReqRow.java | 26 +- .../sql/side/redis/RedisAsyncSideInfo.java | 14 +- .../sql/side/redis/table/RedisSideParser.java | 11 +- .../sql/side/redis/table/RedisSideReqRow.java | 8 +- .../side/redis/table/RedisSideTableInfo.java | 8 +- .../sql/sink/redis/RedisOutputFormat.java | 29 +- .../flink/sql/sink/redis/RedisSink.java | 8 +- .../sql/sink/redis/table/RedisSinkParser.java | 12 +- .../sql/sink/redis/table/RedisTableInfo.java | 8 +- .../CustomerSocketTextStreamFunction.java | 6 +- .../serversocket/ServersocketSource.java | 4 +- .../table/ServersocketSourceParser.java | 8 +- .../table/ServersocketSourceTableInfo.java | 6 +- .../side/sqlserver/SqlserverAllReqRow.java | 20 +- .../side/sqlserver/SqlserverAllSideInfo.java | 10 +- .../side/sqlserver/SqlserverAsyncReqRow.java | 10 +- .../sqlserver/SqlserverAsyncSideInfo.java | 11 +- .../sqlserver/table/SqlserverSideParser.java | 12 +- .../sql/sink/sqlserver/SqlserverSink.java | 6 +- .../sqlserver/table/SqlserverSinkParser.java | 6 +- 256 files changed, 3487 insertions(+), 2621 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 ci/sonar_notify.sh rename core/src/main/java/com/dtstack/flink/sql/outputformat/{DtRichOutputFormat.java => AbstractDtRichOutputFormat.java} (95%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideTableInfo.java => AbstractSideTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/side/{AllReqRow.java => BaseAllReqRow.java} (79%) rename core/src/main/java/com/dtstack/flink/sql/side/{AsyncReqRow.java => BaseAsyncReqRow.java} (89%) rename core/src/main/java/com/dtstack/flink/sql/side/{SideInfo.java => BaseSideInfo.java} (92%) rename core/src/main/java/com/dtstack/flink/sql/side/cache/{AbsSideCache.java => AbstractSideCache.java} (84%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSideTableParser.java => AbstractSideTableParser.java} (68%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsSourceParser.java => AbstractSourceParser.java} (81%) rename core/src/main/java/com/dtstack/flink/sql/table/{SourceTableInfo.java => AbstractSourceTableInfo.java} (97%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfo.java => AbstractTableInfo.java} (98%) rename core/src/main/java/com/dtstack/flink/sql/table/{TableInfoParser.java => AbstractTableInfoParser.java} (83%) rename core/src/main/java/com/dtstack/flink/sql/table/{AbsTableParser.java => AbstractTableParser.java} (88%) rename core/src/main/java/com/dtstack/flink/sql/table/{TargetTableInfo.java => AbstractTargetTableInfo.java} (94%) rename core/src/main/java/com/dtstack/flink/sql/watermarker/{AbsCustomerWaterMarker.java => AbstractCustomerWaterMarker.java} (94%) create mode 100644 docs/kafkaSink.md rename hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/{AbsRowKeyModeDealer.java => AbstractRowKeyModeDealer.java} (87%) rename hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/{ReplaceOperator.java => AbstractReplaceOperator.java} (93%) create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java create mode 100644 kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java rename cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java => kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java (56%) delete mode 100644 mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java delete mode 100644 mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java rename rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/{RdbAllReqRow.java => AbstractRdbAllReqRow.java} (96%) rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/{RdbSink.java => AbstractRdbSink.java} (90%) create mode 100644 rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java rename rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/{UpsertWriter.java => AbstractUpsertWriter.java} (84%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..2952551ff --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,10 @@ +build: + stage: test + script: + - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q + - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.branch.name="v1.8.0_dev" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar + - sh ci/sonar_notify.sh + only: + - v1.8.0_dev + tags: + - dt-insight-engine \ No newline at end of file diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index bc0a0f8cd..3ad8b934e 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -34,10 +34,10 @@ import com.datastax.driver.core.SocketOptions; import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -62,14 +62,12 @@ * * @author xuqianjin */ -public class CassandraAllReqRow extends AllReqRow { +public class CassandraAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 54015343561288219L; private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class); - private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver"; - private static final int CONN_RETRY_NUM = 3; private static final int FETCH_SIZE = 1000; @@ -79,7 +77,7 @@ public class CassandraAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index fa665f9a0..5d95dc9bf 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; @@ -37,16 +37,16 @@ * * @author xuqianjin */ -public class CassandraAllSideInfo extends SideInfo { +public class CassandraAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -8690814317653033557L; - public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; sqlCondition = "select ${selectField} from ${tableName} "; diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index 0173f2d6f..7eee3cb5f 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -37,11 +37,11 @@ import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy; import com.datastax.driver.core.policies.RetryPolicy; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.google.common.base.Function; @@ -67,7 +67,7 @@ * * @author xuqianjin */ -public class CassandraAsyncReqRow extends AsyncReqRow { +public class CassandraAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 6631584128079864735L; @@ -83,7 +83,7 @@ public class CassandraAsyncReqRow extends AsyncReqRow { private transient ListenableFuture session; private transient CassandraSideTableInfo cassandraSideTableInfo; - public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -216,7 +216,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce connCassandraDB(cassandraSideTableInfo); String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING "; - System.out.println("sqlCondition:" + sqlCondition); + LOG.info("sqlCondition:{}" + sqlCondition); ListenableFuture resultSet = Futures.transformAsync(session, new AsyncFunction() { @@ -265,7 +265,6 @@ public void onSuccess(List rows) { public void onFailure(Throwable t) { LOG.error("Failed to retrieve the data: %s%n", t.getMessage()); - System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); resultFuture.completeExceptionally(t); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 3557f0f73..4bee5648b 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; @@ -30,6 +30,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -39,16 +41,18 @@ * * @author xuqianjin */ -public class CassandraAsyncSideInfo extends SideInfo { +public class CassandraAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4403313049809013362L; + private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName()); - public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + + public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -63,9 +67,9 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { } sqlCondition = "select ${selectField} from ${tableName}"; - sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - System.out.println("---------side_exe_sql-----\n" + sqlCondition); + + LOG.info("---------side_exe_sql-----\n{}" + sqlCondition); } diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 62dd753b9..e893e56b0 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -19,18 +19,16 @@ package com.dtstack.flink.sql.side.cassandra.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; -import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -38,7 +36,7 @@ * * @author xuqianjin */ -public class CassandraSideParser extends AbsSideTableParser { +public class CassandraSideParser extends AbstractSideTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; @@ -73,7 +71,7 @@ public CassandraSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo(); cassandraSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraSideTableInfo); @@ -96,9 +94,10 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class CassandraOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = -7994311331389155692L; private static final Logger LOG = LoggerFactory.getLogger(CassandraOutputFormat.class); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index eb7b23b53..26152a7d3 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.cassandra.table.CassandraTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -63,7 +63,7 @@ public CassandraSink() { } @Override - public CassandraSink genStreamSink(TargetTableInfo targetTableInfo) { + public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { CassandraTableInfo cassandraTableInfo = (CassandraTableInfo) targetTableInfo; this.address = cassandraTableInfo.getAddress(); this.tableName = cassandraTableInfo.getTableName(); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java index 4c68e71ae..9ef8639ba 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -33,7 +33,7 @@ * * @author xuqianjin */ -public class CassandraSinkParser extends AbsTableParser { +public class CassandraSinkParser extends AbstractTableParser { public static final String ADDRESS_KEY = "address"; @@ -60,7 +60,7 @@ public class CassandraSinkParser extends AbsTableParser { public static final String POOL_TIMEOUT_MILLIS_KEY = "poolTimeoutMillis"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { CassandraTableInfo cassandraTableInfo = new CassandraTableInfo(); cassandraTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, cassandraTableInfo); diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index c6626c42a..ffb5fa876 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class CassandraTableInfo extends TargetTableInfo { +public class CassandraTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "cassandra"; diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh new file mode 100644 index 000000000..41f8a3c0e --- /dev/null +++ b/ci/sonar_notify.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq + sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) + curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ + -H "Content-Type: application/json" \ + -d "{ + \"msgtype\": \"markdown\", + \"markdown\": { + \"title\":\"sonar代码质量\", + \"text\": \"## sonar代码质量报告: \n +> [sonar地址](http://172.16.100.198:9000/dashboard?id=dt-insight-engine/flinkStreamSQL) \n +> ${sonarreport} \n\" + } + }" \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java index 68c0c7984..c9a0c447b 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java @@ -20,40 +20,37 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; -public class ClickhouseAllReqRow extends RdbAllReqRow { +public class ClickhouseAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String passWord) { + public Connection getConn(String dbUrl, String userName, String passWord) { try { Connection connection ; JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); // ClickHouseProperties contains all properties if (userName == null) { - connection = DriverManager.getConnection(dbURL); + connection = DriverManager.getConnection(dbUrl); } else { - connection = DriverManager.getConnection(dbURL, userName, passWord); + connection = DriverManager.getConnection(dbUrl, userName, passWord); } return connection; } catch (Exception e) { diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java index 973c069b9..43fbeaa56 100644 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -28,7 +28,7 @@ public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index db49e4337..25478c0dd 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -37,7 +37,7 @@ public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -62,7 +62,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); } } diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java index 254561de0..eec5fbe74 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -29,7 +29,7 @@ public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java index 7be387fd8..ab285c37f 100644 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.clickhouse.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -38,8 +38,8 @@ public class ClickhouseSideParser extends RdbSideParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java index bbb6a6fe4..5f7e41f7b 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java @@ -22,14 +22,11 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class ClickhouseSink extends RdbSink implements IStreamSinkGener { +public class ClickhouseSink extends AbstractRdbSink implements IStreamSinkGener { public ClickhouseSink() { super(new ClickhouseDialect()); } @@ -37,7 +34,7 @@ public ClickhouseSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java index 8c3df93d7..5b0f2598f 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.clickhouse.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Map; @@ -30,8 +30,8 @@ public class ClickhouseSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "clickhouse"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); clickhouseTableInfo.setType(CURR_TYPE); return clickhouseTableInfo; } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java index a397036ef..fc5a768c4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -37,7 +37,7 @@ * * @author xuqianjin */ -public class ConsoleOutputFormat extends DtRichOutputFormat { +public class ConsoleOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); @@ -56,7 +56,7 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) throws IOException { - System.out.println("received oriainal data:" + tuple2); + LOG.info("received oriainal data:{}" + tuple2); Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); if (!retract) { diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java index 77a3efea2..6cfdd2d3d 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.console; import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -80,7 +80,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { + public ConsoleSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { return this; } } diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java index e77444bfd..93ed02420 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java @@ -18,13 +18,13 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -32,9 +32,9 @@ * * @author xuqianjin */ -public class ConsoleSinkParser extends AbsTableParser { +public class ConsoleSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); consoleTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, consoleTableInfo); diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java index 4b286c667..6fd7063c6 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.console.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -26,7 +26,7 @@ * * @author xuqianjin */ -public class ConsoleTableInfo extends TargetTableInfo { +public class ConsoleTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "console"; diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java index 8813da619..12bb2a5d4 100644 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java @@ -22,6 +22,7 @@ public class TablePrintUtil { public static final int ALIGN_LEFT = 1;//左对齐 public static final int ALIGN_RIGHT = 2;//右对齐 public static final int ALIGN_CENTER = 3;//居中对齐 + private static final Pattern PATTERN = Pattern.compile("[\u4e00-\u9fa5]"); private int align = ALIGN_CENTER;//默认居中对齐 private boolean equilong = false;//默认不等宽 @@ -58,7 +59,9 @@ public static TablePrintUtil build(String[][] data) { public static TablePrintUtil build(List data) { TablePrintUtil self = new TablePrintUtil(); self.data = new ArrayList<>(); - if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); + if (data.size() <= 0) { + throw new RuntimeException("数据源至少得有一行吧"); + } Object obj = data.get(0); @@ -70,7 +73,9 @@ public static TablePrintUtil build(List data) { int length = ((List) obj).size(); for (Object item : data) { List col = (List) item; - if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); + if (col.size() != length) { + throw new RuntimeException("数据源每列长度必须一致"); + } self.data.add(col.toArray(new String[length])); } } else { @@ -90,7 +95,7 @@ public static TablePrintUtil build(List data) { try { value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - e.printStackTrace(); + LOG.error("", e); } item[j] = value == null ? "null" : value; } @@ -115,7 +120,7 @@ private static List

getColList(Object obj) { Method[] methods = obj.getClass().getMethods(); for (Method m : methods) { StringBuilder getMethodName = new StringBuilder(m.getName()); - if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { + if ("get".equals(getMethodName.substring(0, 3)) && !"getClass".equals(m.getName())) { Col col = new Col(); col.getMethodName = getMethodName.toString(); char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); @@ -134,8 +139,8 @@ private static ListgetColList(Object obj) { * @return */ private int getStringCharLength(String str) { - Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 - Matcher m = p.matcher(str); + //利用正则找到中文 + Matcher m = PATTERN.matcher(str); int count = 0; while (m.find()) { count++; @@ -163,7 +168,9 @@ private int[] getColLengths() { if (equilong) {//如果等宽表格 int max = 0; for (int len : result) { - if (len > max) max = len; + if (len > max) { + max = len; + } } for (int i = 0; i < result.length; i++) { result[i] = max; @@ -213,6 +220,8 @@ public String getTableString() { sb.append(cell); for (int i = 0; i < right + padding; i++) {sb.append(s);} break; + default: + break; } sb.append(v); } diff --git a/core/pom.xml b/core/pom.xml index b17209612..eb470a3c0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ 1.16.0 2.7.9 19.0 - 1.1.7 + 1.0.0-SNAPSHOT @@ -122,13 +122,11 @@ junit 4.12 - - ch.qos.logback - logback-classic - ${logback.version} + com.aiweiergou + tools-logger + ${logger.tool.version} - diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index b54c3feb2..fe925f9a6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -21,14 +21,12 @@ -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - import com.dtstack.flink.sql.exec.ExecuteProcessHelper; import com.dtstack.flink.sql.exec.ParamsInfo; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Date: 2018/6/26 * Company: www.dtstack.com @@ -40,6 +38,7 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index c03f60617..2e62e11ab 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -98,9 +98,9 @@ public static List getClassPath() { - public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) + public static URLClassLoader loadExtraJar(List jarUrlList, URLClassLoader classLoader) throws IllegalAccessException, InvocationTargetException { - for(URL url : jarURLList){ + for(URL url : jarUrlList){ if(url.toString().endsWith(".jar")){ urlClassLoaderAddUrl(classLoader, url); } diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java index 00d1ea0a5..13e1de8c8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java @@ -63,14 +63,21 @@ public class ConfigConstrant { public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir"; public static final String STATE_BACKEND_INCREMENTAL_KEY = "state.backend.incremental"; + public static final String RESTOREENABLE = "restore.enable"; + public static final String LOG_LEVEL_KEY = "logLevel"; // restart plocy - public static final int failureRate = 3; + public static final int FAILUEE_RATE = 3; + + public static final int FAILUEE_INTERVAL = 6; //min + + public static final int DELAY_INTERVAL = 10; //sec + + public static final String FAILUREINTERVAL = "failure.interval"; //min - public static final int failureInterval = 6; //min + public static final String DELAYINTERVAL= "delay.interval"; //sec - public static final int delayInterval = 10; //sec } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 749bbc907..7f3f0019c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -46,11 +46,66 @@ * @author huyifan.zju@163.com */ public enum ColumnType { - STRING, VARCHAR, CHAR, - INT, MEDIUMINT, TINYINT, DATETIME, SMALLINT, BIGINT, - DOUBLE, FLOAT, + /* + * string + */ + STRING, + /** + * varchar + */ + VARCHAR, + /** + * char + */ + CHAR, + /** + * int + */ + INT, + /** + * mediumint + */ + MEDIUMINT, + /** + * tinyint + */ + TINYINT, + /** + * datetime + */ + DATETIME, + /** + * smallint + */ + SMALLINT, + /** + * bigint + */ + BIGINT, + /** + * double + */ + DOUBLE, + /** + * float + */ + FLOAT, + /** + * boolean + */ BOOLEAN, - DATE, TIMESTAMP, DECIMAL; + /** + * date + */ + DATE, + /** + * timestamp + */ + TIMESTAMP, + /** + * decimal + */ + DECIMAL; public static ColumnType fromString(String type) { if(type == null) { diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java index 66160d820..49e352757 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheContentType.java @@ -30,8 +30,17 @@ public enum ECacheContentType { + /** + * 无 + */ MissVal(0), + /** + * 1行 + */ SingleLine(1), + /** + * 多行 + */ MultiLine(2); int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java index 582148c2c..9d5bb5d11 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ECacheType.java @@ -27,7 +27,18 @@ * @author xuchao */ public enum ECacheType { - NONE, LRU, ALL; + /** + * none + */ + NONE, + /** + * lru + */ + LRU, + /** + * all + */ + ALL; public static boolean isValid(String type){ for(ECacheType tmpType : ECacheType.values()){ diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java index 9b01bf052..7d2235f2e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EDatabaseType.java @@ -26,8 +26,17 @@ */ public enum EDatabaseType { + /** + * mysql + */ MYSQL, + /** + * sqlserver + */ SQLSERVER, + /** + * oracle + */ ORACLE, } diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java index 6cb027ac3..439966dd2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java @@ -26,7 +26,13 @@ */ public enum EPluginLoadMode { + /** + * 0:classpath + */ CLASSPATH(0), + /** + * 1:shipfile + */ SHIPFILE(1); private int type; diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java index a8f926175..098cb57fe 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/EStateBackend.java @@ -25,8 +25,17 @@ * @author maqi */ public enum EStateBackend { + /** + * memory + */ MEMORY, + /** + * rockdb + */ ROCKSDB, + /** + * filesystem + */ FILESYSTEM; public static EStateBackend convertFromString(String type) { diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java index ce763ea99..d7b772c6e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.environment; +import com.dtstack.flink.sql.constrant.ConfigConstrant; +import com.dtstack.flink.sql.enums.EStateBackend; +import com.dtstack.flink.sql.util.MathUtil; +import com.dtstack.flink.sql.util.PropertiesUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; @@ -34,13 +40,6 @@ import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.java.StreamTableEnvironment; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.EStateBackend; -import com.dtstack.flink.sql.util.MathUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; - import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -102,11 +101,15 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s } }); - streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) - )); + if(isRestore(confProperties).get()){ + streamEnv.setRestartStrategy(RestartStrategies.failureRateRestart( + ConfigConstrant.FAILUEE_RATE, + Time.of(getFailureInterval(confProperties).get(), TimeUnit.MINUTES), + Time.of(getDelayInterval(confProperties).get(), TimeUnit.SECONDS) + )); + } else { + streamEnv.setRestartStrategy(RestartStrategies.noRestart()); + } // checkpoint config Optional checkpointingEnabled = isCheckpointingEnabled(confProperties); @@ -164,6 +167,20 @@ public static Optional getAutoWatermarkInterval(Properties properties) { return StringUtils.isNotBlank(autoWatermarkInterval) ? Optional.of(Long.valueOf(autoWatermarkInterval)) : Optional.empty(); } + public static Optional isRestore(Properties properties){ + String restoreEnable = properties.getProperty(ConfigConstrant.RESTOREENABLE, "true"); + return Optional.of(Boolean.valueOf(restoreEnable)); + } + + public static Optional getDelayInterval(Properties properties){ + String delayInterval = properties.getProperty(ConfigConstrant.DELAYINTERVAL, "10"); + return Optional.of(Integer.valueOf(delayInterval)); + } + public static Optional getFailureInterval(Properties properties){ + String failureInterval = properties.getProperty(ConfigConstrant.FAILUREINTERVAL, "6"); + return Optional.of(Integer.valueOf(failureInterval)); + } + /** * #ProcessingTime(默认), IngestionTime, EventTime * @param properties @@ -259,6 +276,8 @@ private static Optional createStateBackend(String backendType, Str checkpointDataUriEmptyCheck(checkpointDataUri, backendType); stateBackend = new RocksDBStateBackend(checkpointDataUri, BooleanUtils.toBoolean(backendIncremental)); break; + default: + break; } return stateBackend == null ? Optional.empty() : Optional.of(stateBackend); } @@ -318,14 +337,14 @@ private static void verityTtl(String ttlMintimeStr, String ttlMaxtimeStr) { * @return */ private static Long getTtlTime(Integer timeNumber, String timeUnit) { - if (timeUnit.equalsIgnoreCase("d")) { - return timeNumber * 1000l * 60 * 60 * 24; - } else if (timeUnit.equalsIgnoreCase("h")) { - return timeNumber * 1000l * 60 * 60; - } else if (timeUnit.equalsIgnoreCase("m")) { - return timeNumber * 1000l * 60; - } else if (timeUnit.equalsIgnoreCase("s")) { - return timeNumber * 1000l; + if ("d".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60 * 24; + } else if ("h".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60 * 60; + } else if ("m".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L * 60; + } else if ("s".equalsIgnoreCase(timeUnit)) { + return timeNumber * 1000L; } else { throw new RuntimeException("not support " + timeNumber + timeUnit); } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 3c9f37811..3858dc5c5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,7 +18,13 @@ package com.dtstack.flink.sql.exec; -import com.dtstack.flink.sql.parser.*; +import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; +import com.dtstack.flink.sql.parser.CreateFuncParser; +import com.dtstack.flink.sql.parser.CreateTmpTableParser; +import com.dtstack.flink.sql.parser.FlinkPlanner; +import com.dtstack.flink.sql.parser.InsertSqlParser; +import com.dtstack.flink.sql.parser.SqlParser; +import com.dtstack.flink.sql.parser.SqlTree; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,6 +39,7 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; @@ -42,12 +49,12 @@ import com.dtstack.flink.sql.option.OptionParser; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.side.SideSqlExec; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; @@ -90,11 +97,8 @@ public class ExecuteProcessHelper { public static ParamsInfo parseParams(String[] args) throws Exception { LOG.info("------------program params-------------------------"); - System.out.println("------------program params-------------------------"); Arrays.stream(args).forEach(arg -> LOG.info("{}", arg)); - Arrays.stream(args).forEach(System.out::println); LOG.info("-------------------------------------------"); - System.out.println("----------------------------------------"); OptionParser optionParser = new OptionParser(args); Options options = optionParser.getOptions(); @@ -105,13 +109,14 @@ public static ParamsInfo parseParams(String[] args) throws Exception { String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); String pluginLoadMode = options.getPluginLoadMode(); String deployMode = options.getMode(); + String logLevel = options.getLogLevel(); Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode), "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required"); String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - List jarURList = getExternalJarUrls(options.getAddjar()); + List jarUrlList = getExternalJarUrls(options.getAddjar()); return ParamsInfo.builder() .setSql(sql) @@ -121,7 +126,7 @@ public static ParamsInfo parseParams(String[] args) throws Exception { .setPluginLoadMode(pluginLoadMode) .setDeployMode(deployMode) .setConfProp(confProperties) - .setJarUrlList(jarURList) + .setJarUrlList(jarUrlList) .build(); } @@ -152,7 +157,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf SqlParser.setLocalSqlPluginRoot(paramsInfo.getLocalSqlPluginPath()); SqlTree sqlTree = SqlParser.parseSql(paramsInfo.getSql()); - Map sideTableMap = Maps.newHashMap(); + Map sideTableMap = Maps.newHashMap(); Map registerTableCache = Maps.newHashMap(); //register udf @@ -185,14 +190,14 @@ public static List getExternalJarUrls(String addJarListStr) throws java.io. } return jarUrlList; } - + private static void sqlTranslation(String localSqlPluginPath, - StreamTableEnvironment tableEnv, - SqlTree sqlTree,Map sideTableMap, - Map registerTableCache, - StreamQueryConfig queryConfig) throws Exception { + StreamTableEnvironment tableEnv, + SqlTree sqlTree,Map sideTableMap, + Map registerTableCache, + StreamQueryConfig queryConfig) throws Exception { - SideSqlExec sideSqlExec = new SideSqlExec(); + SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); @@ -224,9 +229,8 @@ private static void sqlTranslation(String localSqlPluginPath, //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); } else { - System.out.println("----------exec sql without dimension join-----------"); - System.out.println("----------real sql exec is--------------------------"); - System.out.println(result.getExecSql()); + LOG.info("----------exec sql without dimension join-----------"); + LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql(), queryConfig); if (LOG.isInfoEnabled()) { LOG.info("exec sql: " + result.getExecSql()); @@ -266,14 +270,14 @@ public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrl * @throws Exception */ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { Set pluginClassPatshSets = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); - for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { + for (AbstractTableInfo tableInfo : sqlTree.getTableInfoMap().values()) { - if (tableInfo instanceof SourceTableInfo) { + if (tableInfo instanceof AbstractSourceTableInfo) { - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; Table table = StreamSourceFactory.getStreamSource(sourceTableInfo, env, tableEnv, localSqlPluginPath); tableEnv.registerTable(sourceTableInfo.getAdaptName(), table); //Note --- parameter conversion function can not be used inside a function of the type of polymerization @@ -304,21 +308,21 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } registerTableCache.put(tableInfo.getName(), regTable); - URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sourceTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractSourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sourceTablePathUrl); - } else if (tableInfo instanceof TargetTableInfo) { + } else if (tableInfo instanceof AbstractTargetTableInfo) { - TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); + TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sinkTablePathUrl); - } else if (tableInfo instanceof SideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; - sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); + } else if (tableInfo instanceof AbstractSideTableInfo) { + String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); - URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); + URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPatshSets.add(sideTablePathUrl); } else { throw new RuntimeException("not support table type:" + tableInfo.getType()); @@ -351,4 +355,12 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti } + public static void setLogLevel(ParamsInfo paramsInfo){ + String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); + if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ + return; + } + ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); + logLevelProcess.process(logLevel); + } } \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index b36dc31eb..27cc7702d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -84,7 +84,6 @@ public Properties getConfProp() { return confProp; } - @Override public String toString() { return "ParamsInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java index 3a5af18b1..8802198a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/SerializationMetricWrapper.java @@ -24,6 +24,7 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; @@ -34,11 +35,11 @@ * author: toutian * create: 2019/12/24 */ -public class SerializationMetricWrapper implements SerializationSchema { +public class SerializationMetricWrapper implements SerializationSchema { private static final long serialVersionUID = 1L; - private SerializationSchema serializationSchema; + private SerializationSchema serializationSchema; private transient RuntimeContext runtimeContext; @@ -47,7 +48,7 @@ public class SerializationMetricWrapper implements SerializationSchema { protected transient Meter dtNumRecordsOutRate; - public SerializationMetricWrapper(SerializationSchema serializationSchema) { + public SerializationMetricWrapper(SerializationSchema serializationSchema) { this.serializationSchema = serializationSchema; } @@ -57,7 +58,7 @@ public void initMetric() { } @Override - public byte[] serialize(Row element) { + public byte[] serialize(CRow element) { beforeSerialize(); byte[] row = serializationSchema.serialize(element); afterSerialize(); @@ -79,7 +80,7 @@ public void setRuntimeContext(RuntimeContext runtimeContext) { this.runtimeContext = runtimeContext; } - public SerializationSchema getSerializationSchema() { + public SerializationSchema getSerializationSchema() { return serializationSchema; } diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 78b579305..5f1b1c6f3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.format.dtnest; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; @@ -28,8 +28,9 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import java.io.IOException; @@ -57,9 +58,9 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -75,7 +76,7 @@ public Row deserialize(byte[] message) throws IOException { try { for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java index 43f599d14..0fd057c5a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java @@ -31,7 +31,10 @@ import java.io.File; import java.io.FileInputStream; import java.net.URLEncoder; +import java.util.stream.Stream; + import org.apache.commons.codec.Charsets; +import org.apache.flink.util.FileUtils; /** @@ -92,19 +95,16 @@ public Options getOptions(){ } public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.ObjectToMap(properties); + Map mapConf = PluginUtil.objectToMap(properties); List args = Lists.newArrayList(); - for(Map.Entry one : mapConf.entrySet()){ + for (Map.Entry one : mapConf.entrySet()) { String key = one.getKey(); Object value = one.getValue(); - if(value == null){ + if (value == null) { continue; - }else if(OPTION_SQL.equalsIgnoreCase(key)){ + } else if (OPTION_SQL.equalsIgnoreCase(key)) { File file = new File(value.toString()); - FileInputStream in = new FileInputStream(file); - byte[] filecontent = new byte[(int) file.length()]; - in.read(filecontent); - String content = new String(filecontent, Charsets.UTF_8.name()); + String content = FileUtils.readFile(file, "UTF-8"); value = URLEncoder.encode(content, Charsets.UTF_8.name()); } args.add("-" + key); @@ -112,9 +112,4 @@ public List getProgramExeArgList() throws Exception { } return args; } - - public static void main(String[] args) throws Exception { - OptionParser OptionParser = new OptionParser(args); - System.out.println(OptionParser.getOptions()); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java similarity index 95% rename from core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java rename to core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java index 1fc40c13b..fbcc86bbd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/outputformat/DtRichOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/outputformat/AbstractDtRichOutputFormat.java @@ -27,7 +27,7 @@ * extend RichOutputFormat with metric 'dtNumRecordsOut', 'dtNumDirtyRecordsOut', 'dtNumRecordsOutRate' * Created by sishu.yss on 2018/11/28. */ -public abstract class DtRichOutputFormat extends RichOutputFormat{ +public abstract class AbstractDtRichOutputFormat extends RichOutputFormat{ public transient Counter outRecords; public transient Counter outDirtyRecords; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index 670d98a7e..fc6a7e16c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -32,18 +32,18 @@ public class CreateFuncParser implements IParser { - private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String FUNC_PATTERN_STR = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; - private static final Pattern funcPattern = Pattern.compile(funcPatternStr); + private static final Pattern FUNC_PATTERN = Pattern.compile(FUNC_PATTERN_STR); @Override public boolean verify(String sql) { - return funcPattern.matcher(sql).find(); + return FUNC_PATTERN.matcher(sql).find(); } @Override public void parseSql(String sql, SqlTree sqlTree) { - Matcher matcher = funcPattern.matcher(sql); + Matcher matcher = FUNC_PATTERN.matcher(sql); if(matcher.find()){ String type = matcher.group(1); String funcName = matcher.group(2); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index a76c1b31a..2afc76c48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.enums.ETableType; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TableInfoParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; @@ -75,7 +75,7 @@ public static SqlTree parseSql(String sql) throws Exception { List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); SqlTree sqlTree = new SqlTree(); - TableInfoParser tableInfoParser = new TableInfoParser(); + AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ if(Strings.isNullOrEmpty(childSql)){ continue; @@ -113,7 +113,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -126,7 +126,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SINK.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } @@ -144,7 +144,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("can't find table " + tableName); } } else { - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + AbstractTableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), createTableResult, LOCAL_SQL_PLUGIN_ROOT); sqlTree.addTableInfo(tableName, tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 1b64b7c68..5252ee022 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Maps; import com.google.common.collect.Lists; @@ -41,7 +41,7 @@ public class SqlTree { private Map preDealTableMap = Maps.newHashMap(); - private Map tableInfoMap = Maps.newLinkedHashMap(); + private Map tableInfoMap = Maps.newLinkedHashMap(); private List execSqlList = Lists.newArrayList(); @@ -89,11 +89,11 @@ public List getTmpSqlList(){ return tmpSqlList; } - public Map getTableInfoMap() { + public Map getTableInfoMap() { return tableInfoMap; } - public void addTableInfo(String tableName, TableInfo tableInfo){ + public void addTableInfo(String tableName, AbstractTableInfo tableInfo){ tableInfoMap.put(tableName, tableInfo); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 32f065123..19dd67f9a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ * @author xuchao */ -public abstract class SideTableInfo extends TableInfo implements Serializable { +public abstract class AbstractSideTableInfo extends AbstractTableInfo implements Serializable { public static final String TARGET_SUFFIX = "Side"; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index 64fd844b1..3e3d0a8fb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -25,13 +25,15 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.dtstack.flink.sql.factory.DTThreadFactory; import org.apache.calcite.sql.JoinType; import java.sql.SQLException; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -41,13 +43,15 @@ * @author xuchao */ -public abstract class AllReqRow extends RichFlatMapFunction implements ISideReqRow { +public abstract class BaseAllReqRow extends RichFlatMapFunction implements ISideReqRow { - protected SideInfo sideInfo; + private static final Logger LOG = LoggerFactory.getLogger(BaseAllReqRow.class); + + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; - public AllReqRow(SideInfo sideInfo){ + public BaseAllReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -60,11 +64,11 @@ public AllReqRow(SideInfo sideInfo){ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); - System.out.println("----- all cacheRef init end-----"); + LOG.info("----- all cacheRef init end-----"); //start reload cache thread - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); - es = Executors.newSingleThreadScheduledExecutor(new DTThreadFactory("cache-all-reload")); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + es = new ScheduledThreadPoolExecutor(1,new DTThreadFactory("cache-all-reload")); es.scheduleAtFixedRate(() -> reloadCache(), sideTableInfo.getCacheTimeout(), sideTableInfo.getCacheTimeout(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java similarity index 89% rename from core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 6f0dce58a..7ac4813a2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.metric.MetricConstant; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.cache.LRUSideCache; import org.apache.calcite.sql.JoinType; @@ -30,14 +30,12 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -47,17 +45,17 @@ * @author xuchao */ -public abstract class AsyncReqRow extends RichAsyncFunction implements ISideReqRow { - private static final Logger LOG = LoggerFactory.getLogger(AsyncReqRow.class); +public abstract class BaseAsyncReqRow extends RichAsyncFunction implements ISideReqRow { + private static final Logger LOG = LoggerFactory.getLogger(BaseAsyncReqRow.class); private static final long serialVersionUID = 2098635244857937717L; private static int TIMEOUT_LOG_FLUSH_NUM = 10; private int timeOutNum = 0; - protected SideInfo sideInfo; + protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; - public AsyncReqRow(SideInfo sideInfo){ + public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -70,12 +68,12 @@ public void open(Configuration parameters) throws Exception { } private void initCache(){ - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ return; } - AbsSideCache sideCache; + AbstractSideCache sideCache; if(ECacheType.LRU.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ sideCache = new LRUSideCache(sideTableInfo); sideInfo.setSideCache(sideCache); @@ -124,10 +122,10 @@ protected void dealCacheData(String key, CacheObj missKeyObj) { @Override public void timeout(CRow input, ResultFuture resultFuture) throws Exception { - //TODO 需要添加数据指标 if(timeOutNum % TIMEOUT_LOG_FLUSH_NUM == 0){ LOG.info("Async function call has timed out. input:{}, timeOutNum:{}",input.toString(), timeOutNum); } + timeOutNum ++; if(timeOutNum > sideInfo.getSideTableInfo().getAsyncTimeoutNumLimit()){ resultFuture.completeExceptionally(new Exception("Async function call timedoutNum beyond limit.")); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java similarity index 92% rename from core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java rename to core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index d8a22e022..732691d0e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class SideInfo implements Serializable{ +public abstract class BaseSideInfo implements Serializable{ protected RowTypeInfo rowTypeInfo; @@ -68,12 +68,12 @@ public abstract class SideInfo implements Serializable{ //key:Returns the value of the position, value: the ref field name​in the side table protected Map sideFieldNameIndex = Maps.newHashMap(); - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - protected AbsSideCache sideCache; + protected AbstractSideCache sideCache; - public SideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo){ + public BaseSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo){ this.rowTypeInfo = rowTypeInfo; this.outFieldInfoList = outFieldInfoList; this.joinType = joinInfo.getJoinType(); @@ -173,7 +173,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } - public abstract void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo); + public abstract void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo); public RowTypeInfo getRowTypeInfo() { return rowTypeInfo; @@ -247,19 +247,19 @@ public void setSideFieldIndex(Map sideFieldIndex) { this.sideFieldIndex = sideFieldIndex; } - public SideTableInfo getSideTableInfo() { + public AbstractSideTableInfo getSideTableInfo() { return sideTableInfo; } - public void setSideTableInfo(SideTableInfo sideTableInfo) { + public void setSideTableInfo(AbstractSideTableInfo sideTableInfo) { this.sideTableInfo = sideTableInfo; } - public AbsSideCache getSideCache() { + public AbstractSideCache getSideCache() { return sideCache; } - public void setSideCache(AbsSideCache sideCache) { + public void setSideCache(AbstractSideCache sideCache) { this.sideCache = sideCache; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 544e8ebb0..d2bb2cb0d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.parser.FlinkPlanner; import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; +import com.esotericsoftware.minlog.Log; import com.google.common.base.Preconditions; import com.google.common.collect.*; import org.apache.calcite.sql.JoinType; @@ -438,18 +439,18 @@ private void extractTemporaryQuery(SqlNode node, String tableAlias, tableRef.put(tbTmp, tableAlias); } - System.out.println("-------build temporary query-----------"); - System.out.println(tmpSelectSql); - System.out.println("---------------------------------------"); + Log.info("-------build temporary query-----------\n{}", tmpSelectSql); + Log.info("---------------------------------------"); }catch (Exception e){ - e.printStackTrace(); + Log.error("", e); throw new RuntimeException(e); } } /** * 抽取上层需用使用到的字段 + * 由于where字段已经抽取到上一层了所以不用查询出来 * @param parentSelectList * @param fromTableNameSet * @return @@ -592,7 +593,6 @@ private void extractSelectField(SqlNode selectNode, } }else if(selectNode.getKind() == CASE){ - System.out.println("selectNode"); SqlCase sqlCase = (SqlCase) selectNode; SqlNodeList whenOperands = sqlCase.getWhenOperands(); SqlNodeList thenOperands = sqlCase.getThenOperands(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index 738e0f84e..d9d79f34e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -80,6 +80,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b switch(identifierSize) { case 1: fieldInfoList.addAll(getAllField(scope)); + break; default: SqlIdentifier tableIdentify = identifier.skipLast(1); JoinScope.ScopeChild scopeChild = scope.getScope(tableIdentify.getSimple()); @@ -99,6 +100,7 @@ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, b fieldInfo.setTypeInformation(type); fieldInfoList.add(fieldInfo); } + break; } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java index 4d6112aa8..fa973508f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SidePredicatesParser.java @@ -46,7 +46,7 @@ * @author maqi */ public class SidePredicatesParser { - public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { + public void fillPredicatesForSideTable(String exeSql, Map sideTableMap) throws SqlParseException { FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); parseSql(sqlNode, sideTableMap, Maps.newHashMap()); @@ -58,7 +58,7 @@ public void fillPredicatesForSideTable(String exeSql, Map * @param sideTableMap * @param tabMapping 谓词属性中别名对应的真实维表名称 */ - private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { + private void parseSql(SqlNode sqlNode, Map sideTableMap, Map tabMapping) { SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind) { case INSERT: @@ -101,10 +101,12 @@ private void parseSql(SqlNode sqlNode, Map sideTableMap, parseSql(unionLeft, sideTableMap, tabMapping); parseSql(unionRight, sideTableMap, tabMapping); break; + default: + break; } } - private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { + private void fillToSideTableInfo(Map sideTableMap, Map tabMapping, List predicateInfoList) { predicateInfoList.stream().filter(info -> sideTableMap.containsKey(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable()))) .map(info -> sideTableMap.get(tabMapping.getOrDefault(info.getOwnerTable(), info.getOwnerTable())).getPredicateInfoes().add(info)) .count(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index f37c3f78c..2296f64c5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -47,7 +47,8 @@ import java.util.Queue; import java.util.Set; -import static org.apache.calcite.sql.SqlKind.*; +import static org.apache.calcite.sql.SqlKind.IDENTIFIER; +import static org.apache.calcite.sql.SqlKind.LITERAL; /** * Parsing sql, obtain execution information dimension table @@ -62,8 +63,6 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { - System.out.println("----------exec original Sql----------"); - System.out.println(exeSql); LOG.info("----------exec original Sql----------"); LOG.info(exeSql); @@ -168,6 +167,8 @@ public Object parseSql(SqlNode sqlNode, case LITERAL: return LITERAL.toString(); + default: + break; } return ""; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index f90138b2a..c41e2264a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -43,24 +43,17 @@ import com.dtstack.flink.sql.util.ParseUtils; import com.dtstack.flink.sql.util.TableUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.*; -import org.apache.calcite.sql.SqlAsOperator; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWithItem; -import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -68,7 +61,6 @@ import java.sql.Timestamp; import java.util.Arrays; -import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -97,7 +89,7 @@ public class SideSqlExec { private Map localTableCache = Maps.newHashMap(); public void exec(String sql, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, @@ -131,11 +123,9 @@ public void exec(String sql, if(pollSqlNode.getKind() == INSERT){ - System.out.println("----------real exec sql-----------" ); - System.out.println(pollSqlNode.toString()); FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString(), queryConfig); if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + pollSqlNode.toString()); + LOG.info("----------real exec sql-----------\n{}", pollSqlNode.toString()); } }else if(pollSqlNode.getKind() == AS){ @@ -166,8 +156,7 @@ public void exec(String sql, } }else if (pollObj instanceof JoinInfo){ - System.out.println("----------exec join info----------"); - System.out.println(pollObj.toString()); + LOG.info("----------exec join info----------\n{}", pollObj.toString()); joinFun(pollObj, localTableCache, sideTableMap, tableEnv); } } @@ -294,7 +283,7 @@ private Table getTableFromCache(Map localTableCache, String table * * @return */ - private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, SideTableInfo sideTableInfo) { + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, AbstractSideTableInfo sideTableInfo) { List conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ return true; @@ -302,7 +291,7 @@ private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, return false; } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { + private List convertPrimaryAlias(AbstractSideTableInfo sideTableInfo) { List res = Lists.newArrayList(); sideTableInfo.getPrimaryKeys().forEach(field -> { res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); @@ -310,7 +299,7 @@ private List convertPrimaryAlias(SideTableInfo sideTableInfo) { return res; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName, AbstractSideTableInfo sideTableInfo){ List sqlNodeList = Lists.newArrayList(); ParseUtils.parseAnd(conditionNode, sqlNodeList); List conditionFields = Lists.newArrayList(); @@ -369,7 +358,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, private void joinFun(Object pollObj, Map localTableCache, - Map sideTableMap, + Map sideTableMap, StreamTableEnvironment tableEnv) throws Exception{ JoinInfo joinInfo = (JoinInfo) pollObj; @@ -385,7 +374,7 @@ private void joinFun(Object pollObj, JoinScope.ScopeChild rightScopeChild = new JoinScope.ScopeChild(); rightScopeChild.setAlias(joinInfo.getRightTableAlias()); rightScopeChild.setTableName(joinInfo.getRightTableName()); - SideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); + AbstractSideTableInfo sideTableInfo = sideTableMap.get(joinInfo.getRightTableName()); if(sideTableInfo == null){ sideTableInfo = sideTableMap.get(joinInfo.getRightTableAlias()); } @@ -394,9 +383,9 @@ private void joinFun(Object pollObj, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } -// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ -// throw new RuntimeException("ON condition must contain all equal fields!!!"); -// } + if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ + throw new RuntimeException("ON condition must contain all equal fields!!!"); + } rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 8417c4519..583e4597b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -21,8 +21,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.AbsTableParser; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.PluginUtil; /** @@ -37,7 +37,7 @@ public class StreamSideFactory { private static final String CURR_TYPE = "side"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(cacheType) ? "all" : "async"; String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); @@ -45,10 +45,10 @@ public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sideParser = cl.loadClass(className); - if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { + if (!AbstractSideTableParser.class.isAssignableFrom(sideParser)) { throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); } - return sideParser.asSubclass(AbsTableParser.class).newInstance(); + return sideParser.asSubclass(AbstractTableParser.class).newInstance(); }); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java similarity index 84% rename from core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java rename to core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java index 757f91600..58832c583 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/AbsSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/AbstractSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; /** * Reason: @@ -30,11 +30,11 @@ * @author xuchao */ -public abstract class AbsSideCache { +public abstract class AbstractSideCache { - protected SideTableInfo sideTableInfo; + protected AbstractSideTableInfo sideTableInfo; - public AbsSideCache(SideTableInfo sideTableInfo){ + public AbstractSideCache(AbstractSideTableInfo sideTableInfo){ this.sideTableInfo = sideTableInfo; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 700e13bb2..2664efd34 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cache; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,11 +33,11 @@ * @author xuchao */ -public class LRUSideCache extends AbsSideCache{ +public class LRUSideCache extends AbstractSideCache { protected transient Cache cache; - public LRUSideCache(SideTableInfo sideTableInfo) { + public LRUSideCache(AbstractSideTableInfo sideTableInfo) { super(sideTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 290804200..3c2010907 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.AsyncDataStream; @@ -49,20 +49,20 @@ public class SideAsyncOperator { private static final String ORDERED = "ordered"; - private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + private static BaseAsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(AsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + cl.loadClass(className).asSubclass(BaseAsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? if (ORDERED.equals(sideTableInfo.getCacheMode())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 5aa810b0f..6b6f9fe1b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -20,10 +20,10 @@ package com.dtstack.flink.sql.side.operator; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; @@ -44,22 +44,22 @@ public class SideWithAllCacheOperator { private static final String OPERATOR_TYPE = "All"; - private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, - JoinInfo joinInfo, List outFieldInfoList, - SideTableInfo sideTableInfo) throws Exception { + private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, + JoinInfo joinInfo, List outFieldInfoList, + AbstractSideTableInfo sideTableInfo) throws Exception { String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(BaseAllReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, AbstractSideTableInfo.class) .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, - List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { - AllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { + BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); return inputStream.flatMap(allReqRow); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java index 3cfd48f1b..e29421369 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/IStreamSinkGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; /** * Reason: @@ -30,5 +30,5 @@ */ public interface IStreamSinkGener { - T genStreamSink(TargetTableInfo targetTableInfo); + T genStreamSink(AbstractTargetTableInfo targetTableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 53460081d..89061db3a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.table.sinks.TableSink; @@ -40,21 +40,21 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; - public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class targetParser = cl.loadClass(className); - if(!AbsTableParser.class.isAssignableFrom(targetParser)){ + if(!AbstractTableParser.class.isAssignableFrom(targetParser)){ throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); } - return targetParser.asSubclass(AbsTableParser.class).newInstance(); + return targetParser.asSubclass(AbstractTableParser.class).newInstance(); }); } - public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + public static TableSink getTableSink(AbstractTargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); diff --git a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java index b8b8f6edc..fde5a235d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/IStreamSourceGener.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -38,6 +38,6 @@ public interface IStreamSourceGener { * @param tableEnv * @return */ - T genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); + T genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv); } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index 1057fb0ed..e0cec1415 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -43,17 +43,17 @@ public class StreamSourceFactory { private static final String DIR_NAME_FORMAT = "%ssource"; - public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + public static AbstractSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { Class sourceParser = cl.loadClass(className); - if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ + if(!AbstractSourceParser.class.isAssignableFrom(sourceParser)){ throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); } - return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); + return sourceParser.asSubclass(AbstractSourceParser.class).newInstance(); }); } @@ -62,7 +62,7 @@ public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) * @param sourceTableInfo * @return */ - public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, + public static Table getStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String sqlRootDir) throws Exception { String sourceTypeStr = sourceTableInfo.getType(); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java similarity index 68% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 403360400..4973a891e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.util.Preconditions; @@ -36,24 +36,24 @@ * @author xuchao */ -public abstract class AbsSideTableParser extends AbsTableParser { +public abstract class AbstractSideTableParser extends AbstractTableParser { private final static String SIDE_SIGN_KEY = "sideSignKey"; private final static Pattern SIDE_TABLE_SIGN = Pattern.compile("(?i)^PERIOD\\s+FOR\\s+SYSTEM_TIME$"); - public AbsSideTableParser() { + public AbstractSideTableParser() { addParserHandler(SIDE_SIGN_KEY, SIDE_TABLE_SIGN, this::dealSideSign); } - private void dealSideSign(Matcher matcher, TableInfo tableInfo){ + private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo){ //FIXME SIDE_TABLE_SIGN current just used as a sign for side table; and do nothing } //Analytical create table attributes ==> Get information cache - protected void parseCacheProp(SideTableInfo sideTableInfo, Map props){ - if(props.containsKey(SideTableInfo.CACHE_KEY.toLowerCase())){ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY.toLowerCase())); + protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map props){ + if(props.containsKey(AbstractSideTableInfo.CACHE_KEY.toLowerCase())){ + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY.toLowerCase())); if(cacheType == null){ return; } @@ -63,65 +63,66 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p } sideTableInfo.setCacheType(cacheType); - if(props.containsKey(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ - Integer cacheSize = MathUtil.getIntegerVal(props.get(SideTableInfo.CACHE_SIZE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())){ + Integer cacheSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CACHE_SIZE_KEY.toLowerCase())); if(cacheSize < 0){ throw new RuntimeException("cache size need > 0."); } sideTableInfo.setCacheSize(cacheSize); } - if(props.containsKey(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ - Long cacheTTLMS = MathUtil.getLongVal(props.get(SideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())){ + Long cacheTTLMS = MathUtil.getLongVal(props.get(AbstractSideTableInfo.CACHE_TTLMS_KEY.toLowerCase())); if(cacheTTLMS < 1000){ throw new RuntimeException("cache time out need > 1000 ms."); } sideTableInfo.setCacheTimeout(cacheTTLMS); } - if(props.containsKey(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ - Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(SideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())){ + Boolean partitionedJoinKey = MathUtil.getBoolean(props.get(AbstractSideTableInfo.PARTITIONED_JOIN_KEY.toLowerCase())); if(partitionedJoinKey){ sideTableInfo.setPartitionedJoin(true); } } - if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())){ + String cachemode = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_MODE_KEY.toLowerCase())); - if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ + if(!"ordered".equalsIgnoreCase(cachemode) && !"unordered".equalsIgnoreCase(cachemode)){ throw new RuntimeException("cachemode must ordered or unordered!"); } sideTableInfo.setCacheMode(cachemode.toLowerCase()); } - if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ + Integer asyncCap = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_CAP_KEY.toLowerCase())); if(asyncCap < 0){ throw new RuntimeException("asyncCapacity size need > 0."); } sideTableInfo.setAsyncCapacity(asyncCap); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ + Integer asyncTimeout = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); if (asyncTimeout<0){ throw new RuntimeException("asyncTimeout size need > 0."); } sideTableInfo.setAsyncTimeout(asyncTimeout); } - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ - Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); + if(props.containsKey(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())){ + Integer asyncTimeoutNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_TIMEOUT_NUM_KEY.toLowerCase())); if (asyncTimeoutNum > 0){ sideTableInfo.setAsyncTimeoutNumLimit(asyncTimeoutNum); } } - if (props.containsKey(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { - Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); + if (props.containsKey(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())) { + Integer asyncPoolSize = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.ASYNC_REQ_POOL_KEY.toLowerCase())); Preconditions.checkArgument(asyncPoolSize > 0 && asyncPoolSize <= 20, "asyncPoolSize size limit (0,20]"); sideTableInfo.setAsyncPoolSize(asyncPoolSize); } + } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java similarity index 81% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java index 745357162..308f5859f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceParser.java @@ -33,7 +33,7 @@ * @author xuchao */ -public abstract class AbsSourceParser extends AbsTableParser { +public abstract class AbstractSourceParser extends AbstractTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; private static final String WATERMARK_KEY = "waterMarkKey"; @@ -43,21 +43,21 @@ public abstract class AbsSourceParser extends AbsTableParser { private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); - public AbsSourceParser() { + public AbstractSourceParser() { addParserHandler(VIRTUAL_KEY, virtualFieldKeyPattern, this::dealVirtualField); addParserHandler(WATERMARK_KEY, waterMarkKeyPattern, this::dealWaterMark); addParserHandler(NOTNULL_KEY, notNullKeyPattern, this::dealNotNull); } - protected void dealVirtualField(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealVirtualField(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String fieldName = matcher.group(2); String expression = matcher.group(1); sourceTableInfo.addVirtualField(fieldName, expression); } - protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ - SourceTableInfo sourceTableInfo = (SourceTableInfo) tableInfo; + protected void dealWaterMark(Matcher matcher, AbstractTableInfo tableInfo){ + AbstractSourceTableInfo sourceTableInfo = (AbstractSourceTableInfo) tableInfo; String eventTimeField = matcher.group(1); //FIXME Temporarily resolve the second parameter row_time_field Integer offset = MathUtil.getIntegerVal(matcher.group(3)); @@ -65,11 +65,11 @@ protected void dealWaterMark(Matcher matcher, TableInfo tableInfo){ sourceTableInfo.setMaxOutOrderness(offset); } - protected void dealNotNull(Matcher matcher, TableInfo tableInfo) { + protected void dealNotNull(Matcher matcher, AbstractTableInfo tableInfo) { String fieldName = matcher.group(1); String fieldType = matcher.group(2); Class fieldClass= dbTypeConvertToJavaType(fieldType); - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(true); tableInfo.addPhysicalMappings(fieldName, fieldName); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java index 9a41fa0a1..0fdc0e911 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSourceTableInfo.java @@ -36,7 +36,7 @@ * @author xuchao */ -public abstract class SourceTableInfo extends TableInfo { +public abstract class AbstractSourceTableInfo extends AbstractTableInfo { public static final String SOURCE_SUFFIX = "Source"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java similarity index 98% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 2fdc297a2..3b514dd48 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -34,7 +34,7 @@ * @author xuchao */ -public abstract class TableInfo implements Serializable { +public abstract class AbstractTableInfo implements Serializable { public static final String PARALLELISM_KEY = "parallelism"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java similarity index 83% rename from core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java index 3c1be25c6..d50e1f23c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfoParser.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.StreamSideFactory; import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; @@ -42,7 +42,7 @@ * @author xuchao */ -public class TableInfoParser { +public class AbstractTableInfoParser { private final static String TYPE_KEY = "type"; @@ -50,16 +50,16 @@ public class TableInfoParser { private final static Pattern SIDE_PATTERN = Pattern.compile(SIDE_TABLE_SIGN); - private Map sourceTableInfoMap = Maps.newConcurrentMap(); + private Map sourceTableInfoMap = Maps.newConcurrentMap(); - private Map targetTableInfoMap = Maps.newConcurrentMap(); + private Map targetTableInfoMap = Maps.newConcurrentMap(); - private Map sideTableInfoMap = Maps.newConcurrentMap(); + private Map sideTableInfoMap = Maps.newConcurrentMap(); //Parsing loaded plugin - public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, - String localPluginRoot) throws Exception { - AbsTableParser absTableParser = null; + public AbstractTableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserResult parserResult, + String localPluginRoot) throws Exception { + AbstractTableParser absTableParser = null; Map props = parserResult.getPropMap(); String type = MathUtil.getString(props.get(TYPE_KEY)); @@ -77,9 +77,9 @@ public TableInfo parseWithTableType(int tableType, CreateTableParser.SqlParserRe sourceTableInfoMap.put(type, absTableParser); } }else{ - String cacheType = MathUtil.getString(props.get(SideTableInfo.CACHE_KEY)); - absTableParser = sideTableInfoMap.get(type + cacheType); + absTableParser = sideTableInfoMap.get(type); if(absTableParser == null){ + String cacheType = MathUtil.getString(props.get(AbstractSideTableInfo.CACHE_KEY)); absTableParser = StreamSideFactory.getSqlParser(type, localPluginRoot, cacheType); sideTableInfoMap.put(type + cacheType, absTableParser); } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java similarity index 88% rename from core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 3e4027b2c..1f210cbf2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -39,7 +39,7 @@ * @author xuchao */ -public abstract class AbsTableParser { +public abstract class AbstractTableParser { private static final String PRIMARY_KEY = "primaryKey"; private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; @@ -54,7 +54,7 @@ public abstract class AbsTableParser { private Map handlerMap = Maps.newHashMap(); - public AbsTableParser() { + public AbstractTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); addParserHandler(NEST_JSON_FIELD_KEY, nestJsonFieldKeyPattern, this::dealNestField); } @@ -63,9 +63,9 @@ protected boolean fieldNameNeedsUpperCase() { return true; } - public abstract TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; + public abstract AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception; - public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ + public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ for(Map.Entry keyPattern : patternMap.entrySet()){ Pattern pattern = keyPattern.getValue(); String key = keyPattern.getKey(); @@ -84,7 +84,7 @@ public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ @@ -112,12 +112,12 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ Class fieldClass = null; - TableInfo.FieldExtraInfo fieldExtraInfo = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new TableInfo.FieldExtraInfo(); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); @@ -133,7 +133,7 @@ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); @@ -145,7 +145,7 @@ public void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ * @param matcher * @param tableInfo */ - protected void dealNestField(Matcher matcher, TableInfo tableInfo) { + protected void dealNestField(Matcher matcher, AbstractTableInfo tableInfo) { String physicalField = matcher.group(1); Preconditions.checkArgument(!physicalFieldFunPattern.matcher(physicalField).find(), "No need to add data types when using functions, The correct way is : strLen(name) as nameSize, "); @@ -154,7 +154,7 @@ protected void dealNestField(Matcher matcher, TableInfo tableInfo) { String mappingField = matcher.group(4); Class fieldClass = dbTypeConvertToJavaType(fieldType); boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java rename to core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java index 0dc9ca77b..dbd99f993 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTargetTableInfo.java @@ -29,7 +29,7 @@ * @author xuchao */ -public abstract class TargetTableInfo extends TableInfo { +public abstract class AbstractTargetTableInfo extends AbstractTableInfo { public static final String TARGET_SUFFIX = "Sink"; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java index db804ea34..fab634cdc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/ITableFieldDealHandler.java @@ -30,5 +30,5 @@ */ public interface ITableFieldDealHandler { - void dealPrimaryKey(Matcher matcher, TableInfo tableInfo); + void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index feebbe3f1..4bb4ff0d8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -86,7 +86,8 @@ public static Class stringConvertClass(String str) { case "decimal": case "decimalunsigned": return BigDecimal.class; - + default: + break; } throw new RuntimeException("不支持 " + str + " 类型"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a0f5291e7..409602baa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -16,22 +16,17 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.Calendar; -import java.util.Date; -import java.util.Locale; -import java.util.SimpleTimeZone; -import java.util.TimeZone; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; import java.util.regex.Pattern; import static java.time.format.DateTimeFormatter.ISO_INSTANT; @@ -39,7 +34,7 @@ /** * - * Reason: TODO ADD REASON(可选) + * 日期工具 * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com * @author sishu.yss @@ -47,20 +42,16 @@ */ public class DateUtil { - static final String timeZone = "GMT+8"; - static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss"; - static final String dateFormat = "yyyy-MM-dd"; - static final String timeFormat = "HH:mm:ss"; - static final SimpleDateFormat datetimeFormatter = new SimpleDateFormat(datetimeFormat); - static final SimpleDateFormat dateFormatter = new SimpleDateFormat(dateFormat); - static final SimpleDateFormat timeFormatter = new SimpleDateFormat(timeFormat); + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); - private static final int MILLIS_PER_SECOND = 1000; + public static java.sql.Date columnToDate(Object column) { if(column instanceof String) { return new java.sql.Date(stringToDate((String)column).getTime()); @@ -84,23 +75,32 @@ public static Date stringToDate(String strDate) { return null; } try { - return datetimeFormatter.parse(strDate); - } catch (ParseException ignored) { + ; + return localDateTimetoDate(LocalDateTime.parse(strDate, DATE_TIME_FORMATTER)); + } catch (DateTimeParseException ignored) { } try { - return dateFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDate.parse(strDate, DATE_FORMATTER).atStartOfDay()); + } catch (DateTimeParseException ignored) { } try { - return timeFormatter.parse(strDate); - } catch (ParseException ignored) { + return localDateTimetoDate(LocalDateTime.of(LocalDate.now(), LocalTime.parse(strDate, TIME_FORMATTER))); + } catch (DateTimeParseException ignored) { } throw new RuntimeException("can't parse date"); } + public static Date localDateTimetoDate(LocalDateTime localDateTime){ + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime dateToLocalDateTime(Date date){ + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + /** * * @@ -130,13 +130,13 @@ public static long getTodayStart(long day) { * @return */ public static long getTodayStart(long day,String scope) { - if(scope.equals("MS")){ - return getTodayStart(day)*1000; - }else if(scope.equals("S")){ - return getTodayStart(day); - }else{ - return getTodayStart(day); - } + if("MS".equals(scope)){ + return getTodayStart(day)*1000; + }else if("S".equals(scope)){ + return getTodayStart(day); + }else{ + return getTodayStart(day); + } } /** @@ -168,15 +168,15 @@ public static long getNextDayStart(long day) { * @return */ public static long getNextDayStart(long day,String scope) { - if(scope.equals("MS")){ - return getNextDayStart(day)*1000; - }else if(scope.equals("S")){ - return getNextDayStart(day); - }else{ - return getNextDayStart(day); - } + if("MS".equals(scope)){ + return getNextDayStart(day)*1000; + }else if("S".equals(scope)){ + return getNextDayStart(day); + }else{ + return getNextDayStart(day); + } } - + /** * @@ -237,7 +237,7 @@ public static long getWeekFirst(long day) { /** * 根据某个日期时间戳秒值，获取所在周在一年中是第几周. - * + * * @param day * @return */ @@ -257,17 +257,17 @@ public static int getWeekOfYear(long day) { */ public static String getYesterdayByString(String day, String inFormat, String outFormat){ try { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay - 1); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; - } catch (ParseException e) { - return null; - } + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay - 1); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; + } catch (ParseException e) { + return null; + } } /** @@ -288,7 +288,7 @@ public static String getTomorrowByString(String day, String inFormat, String out String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - + /** * * @param date @@ -320,7 +320,7 @@ public static String get30DaysBeforeByString(String day, String inFormat, String calendar.set(Calendar.DATE, calendarDay - 30); return new SimpleDateFormat(outFormat).format(calendar.getTime()); } - + /** * * @param day @@ -330,14 +330,14 @@ public static String get30DaysBeforeByString(String day, String inFormat, String * @throws ParseException */ public static String get30DaysLaterByString(String day, String inFormat, String outFormat) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(inFormat); - Date date = sdf.parse(day); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int calendarDay = calendar.get(Calendar.DATE); - calendar.set(Calendar.DATE, calendarDay + 30); - String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); - return dayBefore; + SimpleDateFormat sdf = new SimpleDateFormat(inFormat); + Date date = sdf.parse(day); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int calendarDay = calendar.get(Calendar.DATE); + calendar.set(Calendar.DATE, calendarDay + 30); + String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); + return dayBefore; } @@ -349,7 +349,7 @@ public static String get30DaysLaterByString(String day, String inFormat, String * @return String * @throws ParseException */ - public static String getDateStrTOFormat(String day, String inFormat, String outFormat) throws ParseException { + public static String getDateStrToFormat(String day, String inFormat, String outFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -357,8 +357,8 @@ public static String getDateStrTOFormat(String day, String inFormat, String outF String dayBefore = new SimpleDateFormat(outFormat).format(calendar.getTime()); return dayBefore; } - - public static long getDateMillTOFormat(String day, String inFormat) throws ParseException { + + public static long getDateMillToFormat(String day, String inFormat) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(inFormat); Date date = sdf.parse(day); Calendar calendar = Calendar.getInstance(); @@ -474,79 +474,79 @@ public static long getMillByLastWeekDay() { cal.set(Calendar.MILLISECOND, 0); return cal.getTimeInMillis() / 1000; } - - /** - * @return long - */ + + /** + * @return long + */ public static long getMillByDay(int severalDays,String condition) { - int dateT=0; + int dateT=0; Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getMillToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getMillToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getMillToDay(cal,dateT); - } - return getMillToDay(cal,dateT); - } - + if(condition==null){ + return getMillToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getMillToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getMillToDay(cal,dateT); + } + return getMillToDay(cal,dateT); + } + /** * @return long */ public static long getStampByDay(int severalDays,String condition) { - int dateT=0; - Calendar cal = Calendar.getInstance(); - if(condition==null){ - return getStampToDay(cal,dateT); - } - if(condition.equals("-")){ - dateT = (cal.get(Calendar.DATE) - severalDays); - return getStampToDay(cal,dateT); - } - if(condition.equals("+")){ - dateT = (cal.get(Calendar.DATE) + severalDays); - return getStampToDay(cal,dateT); - } - return getStampToDay(cal,dateT); + int dateT=0; + Calendar cal = Calendar.getInstance(); + if(condition==null){ + return getStampToDay(cal,dateT); + } + if("-".equals(condition)){ + dateT = (cal.get(Calendar.DATE) - severalDays); + return getStampToDay(cal,dateT); + } + if("+".equals(condition)){ + dateT = (cal.get(Calendar.DATE) + severalDays); + return getStampToDay(cal,dateT); + } + return getStampToDay(cal,dateT); } /** * @return long */ public static long getMillByDay(){ - return getMillByDay(0,null); + return getMillByDay(0,null); } - + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getMillToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - return cal.getTimeInMillis()/1000; - } - + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis()/1000; + } + /** * @param cal Calendar - * @param dateT Integer + * @param dateT Integer * @return long */ public static long getStampToDay(Calendar cal,int dateT){ - if(dateT!=0){ - cal.set(Calendar.DATE, dateT); - } - return cal.getTimeInMillis(); + if(dateT!=0){ + cal.set(Calendar.DATE, dateT); + } + return cal.getTimeInMillis(); } public static String getToday() { @@ -568,7 +568,7 @@ public static String getDate(long day, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(cal.getTime()); } - + /** * * @param date @@ -578,21 +578,21 @@ public static String getDate(Date date, String format) { SimpleDateFormat sf = new SimpleDateFormat(format); return sf.format(date); } - - + + /** * * @param day * @param format * @return long - * @throws ParseException + * @throws ParseException */ public static long stringToLong(String day, String format) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - long Date = dateFormat.parse(day).getTime(); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + long date = dateFormat.parse(day).getTime(); + return date; } - + /** * @param day * @param format @@ -600,30 +600,30 @@ public static long stringToLong(String day, String format) throws ParseException * @throws ParseException */ public static Date stringToDate(String day, String format) { - try { - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - Date Date = dateFormat.parse(day); - return Date; - } catch (ParseException e) { - return new Date(); - } + try { + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + Date date = dateFormat.parse(day); + return date; + } catch (ParseException e) { + return new Date(); + } } - - + + /** * long型时间戳转为String型 - * + * * @param day 秒 * @return 格式化后的日期 - * @throws ParseException + * @throws ParseException */ public static String longToString(long day, String format) throws ParseException { - if (("" + day).length() <= 10){ + if (("" + day).length() <= 10){ day=day*1000; } - SimpleDateFormat dateFormat = new SimpleDateFormat(format); - String Date = dateFormat.format(day); - return Date; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); + String date = dateFormat.format(day); + return date; } /** @@ -653,39 +653,39 @@ public static long getMillByNow() { return cal.getTimeInMillis(); } - public static int getWeeksBetweenTwoDates(long startDay, long endDay) { - int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; - if(week<1){ - week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; - } - return week; - } + public static int getWeeksBetweenTwoDates(long startDay, long endDay) { + int week = getWeekOfYear(endDay) - getWeekOfYear(startDay) + 1; + if(week<1){ + week = getWeekOfYear(endDay) + getMaxWeekOfYear(startDay) - getWeekOfYear(startDay) + 1; + } + return week; + } - public static int getMaxWeekOfYear(long startDay) { - Calendar cal = Calendar.getInstance(); + public static int getMaxWeekOfYear(long startDay) { + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(startDay * 1000)); return cal.getMaximum(Calendar.WEEK_OF_YEAR); - } - - public static int getMonthsBetweenTwoDates(long startDay, long endDay) { - int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; - if(month<1){ - month = getMonth(endDay) + 12 - getMonth(startDay) +1; - } - return month; - } - - public static Date parseDate(String dateStr, String pattern){ - SimpleDateFormat sdf = new SimpleDateFormat(); - sdf.applyPattern(pattern); - try { - return sdf.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - - /** + } + + public static int getMonthsBetweenTwoDates(long startDay, long endDay) { + int month = DateUtil.getMonth(endDay) - DateUtil.getMonth(startDay) + 1; + if(month<1){ + month = getMonth(endDay) + 12 - getMonth(startDay) +1; + } + return month; + } + + public static Date parseDate(String dateStr, String pattern){ + SimpleDateFormat sdf = new SimpleDateFormat(); + sdf.applyPattern(pattern); + try { + return sdf.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + + /** * * @param time Long 时间 * @return long @@ -703,7 +703,7 @@ public static long getMinuteStart(long time) { firstDay = cal.getTimeInMillis() / 1000; return firstDay; } - + /** * @param time Long * @return long @@ -732,24 +732,24 @@ public static Date getDateByLong(long time){ date.setTime(time); return date; } - + public static Date parseDate(String dateStr, String pattern, Locale locale){ - SimpleDateFormat df = new SimpleDateFormat( - pattern, locale); - - df.setTimeZone(new SimpleTimeZone(0, "GMT")); - try { - return df.parse(dateStr); - } catch (ParseException e) { - return null; - } - } - + SimpleDateFormat df = new SimpleDateFormat( + pattern, locale); + + df.setTimeZone(new SimpleTimeZone(0, "GMT")); + try { + return df.parse(dateStr); + } catch (ParseException e) { + return null; + } + } + public static String getDate(Date date, String format, Locale locale) { - SimpleDateFormat df = new SimpleDateFormat( - format, locale); - df.setTimeZone(new SimpleTimeZone(0, "GMT")); + SimpleDateFormat df = new SimpleDateFormat( + format, locale); + df.setTimeZone(new SimpleTimeZone(0, "GMT")); return df.format(date); } @@ -777,59 +777,33 @@ public static java.sql.Timestamp columnToTimestamp(Object column) { } public static String dateToString(Date date) { - return dateFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_FORMATTER); } public static String timestampToString(Date date) { - return datetimeFormatter.format(date); + LocalDateTime localDateTime = dateToLocalDateTime(date); + return localDateTime.format(DATE_TIME_FORMATTER); } - public static Timestamp getTimestampFromStr(String timeStr) { if (DATETIME.matcher(timeStr).matches()) { Instant instant = Instant.from(ISO_INSTANT.parse(timeStr)); return new Timestamp(instant.getEpochSecond() * MILLIS_PER_SECOND); - } else { - java.sql.Date date = null; - try { - date = new java.sql.Date(datetimeFormatter.parse(timeStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("getTimestampFromStr error data is " + timeStr); - } - return new Timestamp(date.getTime()); } + return new Timestamp(stringToDate(timeStr).getTime()); } public static java.sql.Date getDateFromStr(String dateStr) { - // 2020-01-01 format if (DATE.matcher(dateStr).matches()) { - // convert from local date to instant Instant instant = LocalDate.parse(dateStr).atTime(LocalTime.of(0, 0, 0, 0)).toInstant(ZoneOffset.UTC); - // calculate the timezone offset in millis int offset = TimeZone.getDefault().getOffset(instant.toEpochMilli()); - // need to remove the offset since time has no TZ component return new java.sql.Date(instant.toEpochMilli() - offset); } else if (DATETIME.matcher(dateStr).matches()) { - // 2020-01-01T12:12:12Z format Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); return new java.sql.Date(instant.toEpochMilli()); - } else { - try { - // 2020-01-01 12:12:12.0 format - return new java.sql.Date(datetimeFormatter.parse(dateStr).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } } - } - - - public static String getStringFromTimestamp(Timestamp timestamp) { - return datetimeFormatter.format(timestamp); - } - - public static String getStringFromDate(java.sql.Date date) { - return dateFormatter.format(date); + return new java.sql.Date(stringToDate(dateStr).getTime()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b2486b1ab..10a6b4f63 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -219,14 +219,14 @@ public static String addJdbcParam(String dbUrl, Map addParams, b return preStr + "?" + sb.toString(); } - public static boolean isJosn(String str){ + public static boolean isJson(String str) { boolean flag = false; - if(StringUtils.isNotBlank(str)){ + if (StringUtils.isNotBlank(str)) { try { - objectMapper.readValue(str,Map.class); + objectMapper.readValue(str, Map.class); flag = true; } catch (Throwable e) { - flag=false; + flag = false; } } return flag; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index fde2f166e..3d66d8a6d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -28,10 +28,10 @@ public class JDBCUtils { private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - public final static String lock_str = "jdbc_lock_str"; + public final static String LOCK_STR = "jdbc_lock_str"; public static void forName(String clazz, ClassLoader classLoader) { - synchronized (lock_str){ + synchronized (LOCK_STR){ try { Class.forName(clazz, true, classLoader); DriverManager.setLoginTimeout(10); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index 23513a912..5aaa40a15 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -227,7 +227,7 @@ public static BigDecimal getBigDecimal(Object obj) { } else if (obj instanceof BigInteger) { return new BigDecimal((BigInteger) obj); } else if (obj instanceof Number) { - return new BigDecimal(((Number) obj).doubleValue()); + return BigDecimal.valueOf(((Number) obj).doubleValue()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to BigDecimal."); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index a63e5a663..8eb743cf0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -109,7 +109,7 @@ public static String getSqlSideClassName(String pluginTypeName, String type, Str return CLASS_PRE_STR + "." + type.toLowerCase() + "." + pluginTypeName + "." + pluginClassName; } - public static Map ObjectToMap(Object obj) throws Exception{ + public static Map objectToMap(Object obj) throws Exception{ return objectMapper.readValue(objectMapper.writeValueAsBytes(obj), Map.class); } @@ -173,8 +173,8 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - classLoader.addURL(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + classLoader.addURL(pluginJarUrl); } } @@ -191,8 +191,8 @@ public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLExcept } for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - urlList.add(pluginJarURL); + URL pluginJarUrl = file.toURI().toURL(); + urlList.add(pluginJarUrl); } return urlList.toArray(new URL[urlList.size()]); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java similarity index 94% rename from core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java rename to core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java index ed8f64288..d75d26a61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java @@ -38,7 +38,7 @@ * @author xuchao */ -public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { +public abstract class AbstractCustomerWaterMarker extends BoundedOutOfOrdernessTimestampExtractor implements RichFunction { private static final long serialVersionUID = 1L; @@ -55,7 +55,7 @@ public abstract class AbsCustomerWaterMarker extends BoundedOutOfOrdernessTim protected TimeZone timezone; - public AbsCustomerWaterMarker(Time maxOutOfOrderness) { + public AbstractCustomerWaterMarker(Time maxOutOfOrderness) { super(maxOutOfOrderness); } diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java index 99415ba61..979b4c3d6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForLong.java @@ -35,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForLong extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForLong extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForLong.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java index b7961e410..05ee3e46e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/CustomerWaterMarkerForTimeStamp.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.util.MathUtil; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -36,7 +35,7 @@ * @author xuchao */ -public class CustomerWaterMarkerForTimeStamp extends AbsCustomerWaterMarker { +public class CustomerWaterMarkerForTimeStamp extends AbstractCustomerWaterMarker { private static final Logger logger = LoggerFactory.getLogger(CustomerWaterMarkerForTimeStamp.class); diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index f3208db03..b0ca0335f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.watermarker; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.base.Strings; @@ -40,7 +40,7 @@ public class WaterMarkerAssigner { - public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ + public boolean checkNeedAssignWaterMarker(AbstractSourceTableInfo tableInfo){ if(Strings.isNullOrEmpty(tableInfo.getEventTimeField())){ return false; } @@ -48,7 +48,7 @@ public boolean checkNeedAssignWaterMarker(SourceTableInfo tableInfo){ return true; } - public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, SourceTableInfo sourceTableInfo){ + public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo typeInfo, AbstractSourceTableInfo sourceTableInfo){ String eventTimeFieldName = sourceTableInfo.getEventTimeField(); @@ -75,7 +75,7 @@ public DataStream assignWaterMarker(DataStream dataStream, RowTypeInfo type TypeInformation fieldType = fieldTypes[pos]; - AbsCustomerWaterMarker waterMarker = null; + AbstractCustomerWaterMarker waterMarker = null; if(fieldType.getTypeClass().isAssignableFrom(Timestamp.class)){ waterMarker = new CustomerWaterMarkerForTimeStamp(Time.milliseconds(maxOutOrderness), pos,timeZone); }else if(fieldType.getTypeClass().isAssignableFrom(Long.class)){ diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java index 546fcbbac..feec8e547 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SidePredicatesParserTest.java @@ -54,7 +54,7 @@ public void testfillPredicatesForSideTable() throws SqlParseException { " MyTable.a='1' and s.d='1' and s.d <> '3' and s.c LIKE '%xx%' and s.c in ('1','2') and s.c between '10' and '23' and s.d is not null\n"; - SideTableInfo sideTableInfo = new SideTableInfo(){ + AbstractSideTableInfo sideTableInfo = new AbstractSideTableInfo(){ @Override public boolean check() { return false; @@ -63,7 +63,7 @@ public boolean check() { sideTableInfo.setName("sideTable"); - Map sideTableMap = new HashMap<>(); + Map sideTableMap = new HashMap<>(); sideTableMap.put("sideTable", sideTableInfo); SidePredicatesParser sidePredicatesParser = new SidePredicatesParser(); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java index a77201b80..b354a6269 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllReqRow.java @@ -20,10 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.google.common.collect.Maps; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * Date: 2019/11/20 @@ -40,21 +37,21 @@ * @author xiuzhu */ -public class Db2AllReqRow extends RdbAllReqRow { +public class Db2AllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(Db2AllReqRow.class); private static final String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(DB2_DRIVER); - Connection conn = DriverManager.getConnection(dbURL, userName, password); + Connection conn = DriverManager.getConnection(dbUrl, userName, password); return conn; } catch (Exception e) { LOG.error("", e); diff --git a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java index b40cfd772..4ce01bcc8 100644 --- a/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java +++ b/db2/db2-side/db2-all-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; @@ -37,7 +37,7 @@ */ public class Db2AllSideInfo extends RdbAllSideInfo { - public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index ed9fb025b..b190973e3 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -45,7 +45,7 @@ public class Db2AsyncReqRow extends RdbAsyncReqRow { private final static String DB2_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Db2AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java index dbb8230e7..91418a1d0 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; @@ -37,7 +37,7 @@ */ public class Db2AsyncSideInfo extends RdbAsyncSideInfo { - public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Db2AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java index db8a07dc6..96be4ec15 100644 --- a/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java +++ b/db2/db2-side/db2-side-core/src/main/java/com/dtstack/flink/sql/side/db2/table/Db2SideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.db2.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class Db2SideParser extends RdbSideParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java index 573a0896a..09844d1a3 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/DbSink.java @@ -1,13 +1,10 @@ package com.dtstack.flink.sql.sink.db; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import java.util.List; -import java.util.Map; - -public class DbSink extends RdbSink { +public class DbSink extends AbstractRdbSink { public DbSink() { super(new DbDialect()); @@ -15,7 +12,7 @@ public DbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java index ba11aabf3..3e823ab20 100644 --- a/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java +++ b/db2/db2-sink/src/main/java/com/dtstack/flink/sql/sink/db/table/DbSinkParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.sink.db.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -10,8 +10,8 @@ public class DbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "db2"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); tableInfo.setType(CURR_TYPE); return tableInfo; } diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md new file mode 100644 index 000000000..5d7c7b2a7 --- /dev/null +++ b/docs/kafkaSink.md @@ -0,0 +1,223 @@ +## 1.格式： +``` +CREATE TABLE tableName( + colName colType, + ... + function(colNameX) AS aliasName, + WATERMARK FOR colName AS withOffset( colName , delayTime ) + )WITH( + type ='kafka11', + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + groupId='test', + parallelism ='parllNum', + ); +``` + +## 2.支持的版本 + kafka09,kafka10,kafka11及以上版本 + **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + +## 3.表结构定义 + +|参数名称|含义| +|----|---| +| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| +| colName | 列名称| +| colType | 列类型 [colType支持的类型](colType.md)| + +## 4.参数： + +|参数名称|含义|是否必填|默认值| +|----|---|---|---| +|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| +|groupId | 需要读取的 groupId 名称|否|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|parallelism | 并行度设置|否|1| +|partitionKeys | 用来分区的字段|否|| +|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| +|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| +|fieldDelimiter | csv数据分隔符|否| , | + + +**kafka相关参数可以自定义，使用kafka.开头即可。** +``` +kafka.consumer.id +kafka.socket.timeout.ms +kafka.fetch.message.max.bytes +kafka.num.consumer.fetchers +kafka.auto.commit.enable +kafka.auto.commit.interval.ms +kafka.queued.max.message.chunks +kafka.rebalance.max.retries +kafka.fetch.min.bytes +kafka.fetch.wait.max.ms +kafka.rebalance.backoff.ms +kafka.refresh.leader.backoff.ms +kafka.consumer.timeout.ms +kafka.exclude.internal.topics +kafka.partition.assignment.strategy +kafka.client.id +kafka.zookeeper.session.timeout.ms +kafka.zookeeper.connection.timeout.ms +kafka.zookeeper.sync.time.ms +kafka.offsets.storage +kafka.offsets.channel.backoff.ms +kafka.offsets.channel.socket.timeout.ms +kafka.offsets.commit.max.retries +kafka.dual.commit.enabled +kafka.partition.assignment.strategy +kafka.socket.receive.buffer.bytes +kafka.fetch.min.bytes +``` + +## 5.样例： + +### json格式： +``` +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + partitionKeys = 'channel,pv', + updateMode='upsert' + ); + +upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} +append模式下发的数据格式：{"channel":"zs","pv":"330"} + +``` + +### avro格式： + +如果updateMode='upsert',schemaInfo需要包含retract属性信息。 + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + --type='console' + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='1', + updateMode='upsert', + sinkdatatype = 'avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} + ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, + {"name":"retract","type":"boolean"}]}' + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` +### csv格式： + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='2', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + updateMode='upsert', + sinkdatatype = 'csv', + fieldDelimiter='*' + + + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a +``` diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 035e39ffb..95a60a3ff 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -1,6 +1,5 @@ ## 1.格式： ``` -数据现在支持json格式{"xx":"bb","cc":"dd"} CREATE TABLE tableName( colName colType, @@ -15,9 +14,8 @@ CREATE TABLE tableName( topic ='topicName', groupId='test', parallelism ='parllNum', - --timezone='America/Los_Angeles', timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 + sourcedatatype ='dt_nest' #可不设置 ); ``` @@ -47,8 +45,10 @@ CREATE TABLE tableName( |topicIsPattern | topic是否是正则表达式格式(true|false) |否| false |offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| -|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' +|sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| +|schemaInfo | avro类型使用的schema信息|否|| +|fieldDelimiter |csv类型使用的数据分隔符|否| | | +|timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` kafka.consumer.id @@ -169,24 +169,10 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` -# 二、csv格式数据源 -根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 -## 1.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置 |否|1| -|sourcedatatype | 数据类型|是 |csv| -|fielddelimiter | 字段分隔符|是 || -|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| -**kafka相关参数可以自定义，使用kafka.开头即可。** -## 2.样例： +## 7.csv格式数据源 + + ``` CREATE TABLE MyTable( name varchar, @@ -203,186 +189,28 @@ CREATE TABLE MyTable( --topic ='mqTest.*', --topicIsPattern='true' parallelism ='1', - sourcedatatype ='csv', - fielddelimiter ='\|', - lengthcheckpolicy = 'PAD' + sourceDatatype ='csv' ); ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|offsetReset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** +## 8.avro格式数据源 -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** ``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false', + kafka.group.id='mqTest', + sourceDataType ='avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"},{"name":"pv","type":"string"}]}' ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** + ``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` + diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 90ffa2582..62d2d8698 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -40,8 +40,6 @@ |----|---|---|----| | type |表明 输出表类型 mongo|是|| | address | 连接mongo数据库 jdbcUrl |是|| - | userName | mongo连接用户名|否|| - | password | mongo连接密码|否|| | tableName | mongo表名称|是|| | database | mongo表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| @@ -64,7 +62,8 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index 9900dea12..fdfe93d22 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -33,8 +33,6 @@ CREATE TABLE tableName( |----|----|----|----| |type |表明 输出表类型 mongo|是|| |address | 连接mongo数据库 jdbcUrl |是|| -|userName | mongo连接用户名|否|| -|password | mongo连接密码|否|| |tableName | mongo表名称|是|| |database | mongo表名称|是|| |parallelism | 并行度设置|否|1| @@ -46,9 +44,8 @@ CREATE TABLE MyResult( pv VARCHAR )WITH( type ='mongo', - address ='172.21.32.1:27017,172.21.32.1:27017', - userName ='dtstack', - password ='abc123', + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] + address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='pv', parallelism ='1' diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 491d1af27..5ca81c5ed 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,7 +34,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,7 +164,7 @@ public void setBulkFlushMaxActions(int bulkFlushMaxActions) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { ElasticsearchTableInfo elasticsearchTableInfo = (ElasticsearchTableInfo) targetTableInfo; esTableInfo = elasticsearchTableInfo; clusterName = elasticsearchTableInfo.getClusterName(); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 93682de46..7988e597c 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,7 +32,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES_ADDRESS = "address"; @@ -56,7 +56,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 9681742c2..f254d5b4e 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -33,7 +33,7 @@ * @author sishu.yss * @Company: www.dtstack.com */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index 648f09fba..b8cf2c46a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -18,24 +18,22 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; import com.dtstack.flink.sql.side.elasticsearch6.util.SwitchUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -49,7 +47,6 @@ import java.io.IOException; import java.io.Serializable; -import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; import java.util.List; @@ -60,7 +57,7 @@ * @author yinxi * @date 2020/1/13 - 1:00 */ -public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { +public class Elasticsearch6AllReqRow extends BaseAllReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AllReqRow.class); @@ -70,7 +67,7 @@ public class Elasticsearch6AllReqRow extends AllReqRow implements Serializable { private SearchRequest searchRequest; private BoolQueryBuilder boolQueryBuilder; - public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java index bf21b3ca8..d353a583d 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllSideInfo.java @@ -18,12 +18,13 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; + import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; @@ -35,15 +36,15 @@ * @author yinxi * @date 2020/1/13 - 1:01 */ -public class Elasticsearch6AllSideInfo extends SideInfo { +public class Elasticsearch6AllSideInfo extends BaseSideInfo { - public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index f67177fe4..1f4bd1bf1 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -18,6 +18,12 @@ package com.dtstack.flink.sql.side.elasticsearch6; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.PredicateInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -26,7 +32,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import com.dtstack.flink.sql.side.elasticsearch6.util.Es6Util; @@ -57,14 +62,14 @@ * @author yinxi * @date 2020/2/13 - 13:10 */ -public class Elasticsearch6AsyncReqRow extends AsyncReqRow implements Serializable { +public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); - public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); SqlNode conditionNode = joinInfo.getCondition(); ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java index 7b3a2f9e7..4118988ab 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncSideInfo.java @@ -18,18 +18,17 @@ package com.dtstack.flink.sql.side.elasticsearch6; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; +import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -37,15 +36,15 @@ * @author yinxi * @date 2020/2/13 - 13:09 */ -public class Elasticsearch6AsyncSideInfo extends SideInfo { +public class Elasticsearch6AsyncSideInfo extends BaseSideInfo { - public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public Elasticsearch6AsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java index 13dfe2995..1b39bbf0f 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; import com.dtstack.flink.sql.side.elasticsearch6.util.ClassUtil; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -30,7 +30,7 @@ * @author yinxi * @date 2020/1/13 - 1:07 */ -public class Elasticsearch6SideParser extends AbsSideTableParser { +public class Elasticsearch6SideParser extends AbstractSideTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { Elasticsearch6SideTableInfo elasticsearch6SideTableInfo = new Elasticsearch6SideTableInfo(); elasticsearch6SideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearch6SideTableInfo); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java index 0afe2d59e..a01c90a53 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/table/Elasticsearch6SideTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.elasticsearch6.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -26,7 +26,7 @@ * @author yinxi * @date 2020/1/13 - 15:00 */ -public class Elasticsearch6SideTableInfo extends SideTableInfo { +public class Elasticsearch6SideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index cc591789e..3d9e4d62a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.elasticsearch6.table.Elasticsearch6SideTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; @@ -98,7 +98,7 @@ public static RestHighLevelClient getClient(String esAddress, Boolean isAuthMesh } // add index and type to search request - public static SearchRequest setSearchRequest(SideInfo sideInfo) { + public static SearchRequest setSearchRequest(BaseSideInfo sideInfo) { SearchRequest searchRequest = new SearchRequest(); Elasticsearch6SideTableInfo tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); // determine existence of index @@ -129,7 +129,7 @@ public static SearchRequest setSearchRequest(SideInfo sideInfo) { } // build where cause - public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { + public static BoolQueryBuilder setPredicateclause(BaseSideInfo sideInfo) { BoolQueryBuilder boolQueryBuilder = null; List predicateInfoes = sideInfo.getSideTableInfo().getPredicateInfoes(); @@ -144,7 +144,7 @@ public static BoolQueryBuilder setPredicateclause(SideInfo sideInfo) { } // build filter condition - public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, SideInfo sideInfo) { + public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBuilder, PredicateInfo info, BaseSideInfo sideInfo) { switch (info.getOperatorKind()) { case "IN": return boolQueryBuilder.must(QueryBuilders.termsQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition()))); @@ -202,7 +202,7 @@ public static String[] removeSpaceAndApostrophe(String str) { } // prevent word segmentation - public static String textConvertToKeyword(String fieldName, SideInfo sideInfo) { + public static String textConvertToKeyword(String fieldName, BaseSideInfo sideInfo) { String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); String fieldType = sideFieldTypes[fieldIndex]; diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 2646c50e9..b7d9de6fc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,7 +32,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.elasticsearch.table.ElasticsearchTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; @@ -128,7 +128,7 @@ public void emitDataStream(DataStream> dataStream) { } @Override - public ElasticsearchSink genStreamSink(TargetTableInfo targetTableInfo) { + public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { esTableInfo = (ElasticsearchTableInfo) targetTableInfo; clusterName = esTableInfo.getClusterName(); index = esTableInfo.getIndex(); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 8902b953c..22c2b72bc 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchSinkParser extends AbsTableParser { +public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES6_ADDRESS = "address"; @@ -55,7 +55,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ElasticsearchTableInfo elasticsearchTableInfo = new ElasticsearchTableInfo(); elasticsearchTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, elasticsearchTableInfo); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index 3cc3dd9ff..d0eef18f0 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -29,7 +29,7 @@ * @author yinxi * @date 2020/1/9 - 15:06 */ -public class ElasticsearchTableInfo extends TargetTableInfo { +public class ElasticsearchTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "elasticsearch6"; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 11380eea6..ed2931fff 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -45,7 +45,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; -public class HbaseAllReqRow extends AllReqRow { +public class HbaseAllReqRow extends BaseAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(HbaseAllReqRow.class); @@ -55,7 +55,7 @@ public class HbaseAllReqRow extends AllReqRow { private AtomicReference>> cacheRef = new AtomicReference<>(); - public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -134,7 +134,7 @@ public void flatMap(CRow input, Collector out) throws Exception { Map cacheList = null; - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; if (hbaseSideTableInfo.isPreRowKey()) { for (Map.Entry> entry : cacheRef.get().entrySet()) { @@ -153,7 +153,7 @@ public void flatMap(CRow input, Collector out) throws Exception { } private void loadData(Map> tmpCache) throws SQLException { - SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); @@ -182,9 +182,17 @@ private void loadData(Map> tmpCache) throws SQLExcep LOG.error("", e); } finally { try { - conn.close(); - table.close(); - resultScanner.close(); + if (null != conn && !conn.isClosed()) { + conn.close(); + } + + if (null != table) { + table.close(); + } + + if (null != resultScanner) { + resultScanner.close(); + } } catch (IOException e) { LOG.error("", e); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index ea51f46e4..663c2927b 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -31,16 +31,16 @@ import java.util.List; -public class HbaseAllSideInfo extends SideInfo { +public class HbaseAllSideInfo extends BaseSideInfo { private RowKeyBuilder rowKeyBuilder; - public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 251b88034..24ee01786 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -21,12 +21,12 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; -import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbsRowKeyModeDealer; +import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; @@ -58,7 +58,7 @@ * @author xuchao */ -public class HbaseAsyncReqRow extends AsyncReqRow { +public class HbaseAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; @@ -73,13 +73,13 @@ public class HbaseAsyncReqRow extends AsyncReqRow { private transient HBaseClient hBaseClient; - private transient AbsRowKeyModeDealer rowKeyMode; + private transient AbstractRowKeyModeDealer rowKeyMode; private String tableName; private String[] colNames; - public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); tableName = ((HbaseSideTableInfo)sideTableInfo).getTableName(); @@ -89,7 +89,7 @@ public HbaseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List colRefType; - public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public HbaseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { rowKeyBuilder = new RowKeyBuilder(); if(sideTableInfo.getPrimaryKeys().size() < 1){ throw new RuntimeException("Primary key dimension table must be filled"); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java similarity index 87% rename from hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java rename to hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 1506bc440..90ee289bd 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase.rowkeydealer; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import org.apache.calcite.sql.JoinType; import com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -41,7 +41,7 @@ * @author xuchao */ -public abstract class AbsRowKeyModeDealer { +public abstract class AbstractRowKeyModeDealer { protected Map colRefType; @@ -60,9 +60,9 @@ public abstract class AbsRowKeyModeDealer { protected Map sideFieldIndex = Maps.newHashMap(); - public AbsRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, - boolean openCache, JoinType joinType, List outFieldInfoList, - Map inFieldIndex, Map sideFieldIndex){ + public AbstractRowKeyModeDealer(Map colRefType, String[] colNames, HBaseClient hBaseClient, + boolean openCache, JoinType joinType, List outFieldInfoList, + Map inFieldIndex, Map sideFieldIndex){ this.colRefType = colRefType; this.colNames = colNames; this.hBaseClient = hBaseClient; @@ -111,5 +111,5 @@ protected Row fillData(Row input, Object sideInput){ } public abstract void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache); + AbstractSideCache sideCache); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index a41ffe916..dcd50131d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,7 +54,7 @@ * @author xuchao */ -public class PreRowKeyModeDealerDealer extends AbsRowKeyModeDealer { +public class PreRowKeyModeDealerDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(PreRowKeyModeDealerDealer.class); @@ -67,7 +66,7 @@ public PreRowKeyModeDealerDealer(Map colRefType, String[] colNam @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache) { + AbstractSideCache sideCache) { Scanner prefixScanner = hBaseClient.newScanner(tableName); ScanFilter scanFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.UTF8(rowKeyStr))); prefixScanner.setFilter(scanFilter); @@ -80,7 +79,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF } - private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbsSideCache sideCache) { + private String dealOneRow(ArrayList> args, String rowKeyStr, CRow input, ResultFuture resultFuture, AbstractSideCache sideCache) { if(args == null || args.size() == 0){ dealMissKey(input, resultFuture); if (openCache) { @@ -111,8 +110,7 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, for (String key : colNames) { Object val = sideMap.get(key); if (val == null) { - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index b20c316db..41208c7f3 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.cache.AbsSideCache; +import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; @@ -49,7 +49,7 @@ * @author xuchao */ -public class RowKeyEqualModeDealer extends AbsRowKeyModeDealer { +public class RowKeyEqualModeDealer extends AbstractRowKeyModeDealer { private static final Logger LOG = LoggerFactory.getLogger(RowKeyEqualModeDealer.class); @@ -62,7 +62,7 @@ public RowKeyEqualModeDealer(Map colRefType, String[] colNames, @Override public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultFuture resultFuture, - AbsSideCache sideCache){ + AbstractSideCache sideCache){ //TODO 是否有查询多个col family 和多个col的方法 GetRequest getRequest = new GetRequest(tableName, rowKeyStr); hBaseClient.get(getRequest).addCallbacks(arg -> { @@ -86,8 +86,7 @@ public void asyncGetData(String tableName, String rowKeyStr, CRow input, ResultF for(String key : colNames){ Object val = sideMap.get(key); if(val == null){ - System.out.println("can't get data with column " + key); - LOG.error("can't get data with column " + key); + LOG.error("can't get data with column {}", key); } sideVal.add(val); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java similarity index 93% rename from hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java rename to hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java index c3c9b5cfe..c75fa43c1 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/AbstractReplaceOperator.java @@ -31,11 +31,11 @@ * @author xuchao */ -public abstract class ReplaceOperator { +public abstract class AbstractReplaceOperator { private EReplaceOpType opType; - public ReplaceOperator(EReplaceOpType opType){ + public AbstractReplaceOperator(EReplaceOpType opType){ this.opType = opType; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java index af5e24dd6..ab826c98f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/Md5ReplaceOperator.java @@ -30,7 +30,7 @@ * @author xuchao */ -public class Md5ReplaceOperator extends ReplaceOperator{ +public class Md5ReplaceOperator extends AbstractReplaceOperator { public Md5ReplaceOperator(EReplaceOpType opType) { super(opType); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java index 834edf9ae..ac9a3b87c 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceOpType.java @@ -28,5 +28,12 @@ */ public enum EReplaceOpType { - NO_FUNC, MD5_FUNC; + /** + * 没有func + */ + NO_FUNC, + /** + * md5 func + */ + MD5_FUNC; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java index f4b039ef8..bb4b1556b 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/enums/EReplaceType.java @@ -28,7 +28,16 @@ * @author xuchao */ public enum EReplaceType { + /** + * 参数 + */ PARAM, + /** + * 函数 + */ FUNC, + /** + * 常量 + */ CONSTANT; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 7b627da4d..cf44c181f 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -20,15 +20,15 @@ package com.dtstack.flink.sql.side.hbase.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * hbase field information must include the definition of an alias -> sql which does not allow ":" @@ -37,7 +37,7 @@ * @author xuchao */ -public class HbaseSideParser extends AbsSideTableParser { +public class HbaseSideParser extends AbstractSideTableParser { private final static String FIELD_KEY = "fieldKey"; @@ -58,7 +58,7 @@ public HbaseSideParser() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseSideTableInfo hbaseTableInfo = new HbaseSideTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); @@ -77,7 +77,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map { +public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(HbaseOutputFormat.class); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 447b02921..9308725a2 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.hbase.table.HbaseTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,7 +30,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.RetractStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -59,7 +58,7 @@ public HbaseSink() { } @Override - public HbaseSink genStreamSink(TargetTableInfo targetTableInfo) { + public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { HbaseTableInfo hbaseTableInfo = (HbaseTableInfo) targetTableInfo; this.zookeeperQuorum = hbaseTableInfo.getHost(); this.port = hbaseTableInfo.getPort(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 064b13e69..8d8996a23 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; import org.apache.commons.lang3.StringUtils; @@ -32,14 +32,14 @@ import java.util.List; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Date: 2018/09/14 * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseSinkParser extends AbsTableParser { +public class HbaseSinkParser extends AbstractTableParser { public static final String HBASE_ZOOKEEPER_QUORUM = "zookeeperQuorum"; @@ -60,7 +60,7 @@ protected boolean fieldNameNeedsUpperCase() { } @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { HbaseTableInfo hbaseTableInfo = new HbaseTableInfo(); hbaseTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, hbaseTableInfo); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 610eb5c30..62f7c9936 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.sink.hbase.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import java.util.Map; @@ -30,7 +30,7 @@ * Company: www.dtstack.com * @author sishu.yss */ -public class HbaseTableInfo extends TargetTableInfo { +public class HbaseTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "hbase"; diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 9cd8c9194..10938308a 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -20,12 +20,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; -import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +34,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.util.List; -import java.util.Map; /** * side operator with cache for all(period reload) @@ -46,7 +43,7 @@ * @author xiuzhu */ -public class ImpalaAllReqRow extends RdbAllReqRow { +public class ImpalaAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -56,13 +53,13 @@ public class ImpalaAllReqRow extends RdbAllReqRow { private ImpalaSideTableInfo impalaSideTableInfo; - public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.impalaSideTableInfo = (ImpalaSideTableInfo) sideTableInfo; } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Connection connection ; String url = getUrl(); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java index d40c5f48c..a5e643967 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllSideInfo.java @@ -20,10 +20,9 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -32,7 +31,7 @@ public class ImpalaAllSideInfo extends RdbAllSideInfo { - public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -61,7 +60,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 2b76ec26c..b5e0ed5fb 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import io.vertx.core.Vertx; @@ -50,7 +50,7 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; - public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -66,7 +66,10 @@ public void open(Configuration parameters) throws Exception { .put("provider_class", DT_PROVIDER_CLASS) .put("idle_connection_test_period", 300) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) - .put("max_idle_time", 600); + .put("max_idle_time", 600) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); System.setProperty("vertx.disableFileCPResolving", "true"); @@ -75,8 +78,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(impalaSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - - setRdbSQLClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java index 53559b9a7..14fbf0ed3 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -38,7 +38,7 @@ public class ImpalaAsyncSideInfo extends RdbAsyncSideInfo { - public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public ImpalaAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -69,7 +69,7 @@ private String buildPartitionCondition(ImpalaSideTableInfo impalaSideTableInfo) private String getPartitionVaule(String fieldType, List values) { String partitionVaule = values.stream().map(val -> { - return (fieldType.toLowerCase().equals("string") || fieldType.toLowerCase().equals("varchar")) ? "'" + val + "'" : val.toString(); + return ("string".equals(fieldType.toLowerCase()) || "varchar".equals(fieldType.toLowerCase())) ? "'" + val + "'" : val.toString(); }).collect(Collectors.joining(" , ")).toString(); return partitionVaule; diff --git a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java index b6a54ca3f..2f1d73be7 100644 --- a/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java +++ b/impala/impala-side/impala-side-core/src/main/java/com/dtstack/flink/sql/side/impala/table/ImpalaSideParser.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.impala.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; @@ -48,7 +48,7 @@ public class ImpalaSideParser extends RdbSideParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaSideTableInfo impalaSideTableInfo = new ImpalaSideTableInfo(); impalaSideTableInfo.setType(CURR_TYPE); impalaSideTableInfo.setName(tableName); @@ -147,6 +147,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 41fa00926..cd1e1e945 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -21,9 +21,9 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -36,7 +36,7 @@ * @author xiuzhu */ -public class ImpalaSink extends RdbSink implements IStreamSinkGener { +public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { private ImpalaTableInfo impalaTableInfo; @@ -47,7 +47,7 @@ public ImpalaSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(getImpalaJdbcUrl()) + .setDbUrl(getImpalaJdbcUrl()) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) @@ -70,8 +70,8 @@ public JDBCUpsertOutputFormat getOutputFormat() { public String getImpalaJdbcUrl() { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbURL; - StringBuffer urlBuffer = new StringBuffer(dbURL); + String newUrl = dbUrl; + StringBuffer urlBuffer = new StringBuffer(dbUrl); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { @@ -121,7 +121,7 @@ public String getImpalaJdbcUrl() { } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { super.genStreamSink(targetTableInfo); this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; return this; diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 4921f5e51..5b6fdeafe 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -19,12 +19,11 @@ package com.dtstack.flink.sql.sink.impala.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -42,7 +41,7 @@ public class ImpalaSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "impala"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); @@ -117,6 +116,8 @@ public Class dbTypeConvertToJavaType(String fieldType) { return String.class; case "timestamp": return Timestamp.class; + default: + break; } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 88c2ca939..9958a2544 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -19,16 +19,16 @@ import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.AvroCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.CsvCRowSerializationSchema; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.formats.avro.AvroRowSerializationSchema; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.util.Optional; import java.util.Properties; @@ -51,42 +51,37 @@ public abstract class AbstractKafkaProducerFactory { * @param partitioner * @return */ - public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); + public abstract RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys); - protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - return new SerializationMetricWrapper(createSerializationSchema(kafkaSinkTableInfo, typeInformation)); + protected SerializationMetricWrapper createSerializationMetricWrapper(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = createSerializationSchema(kafkaSinkTableInfo, typeInformation); + return new SerializationMetricWrapper(serializationSchema); } - private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { - SerializationSchema serializationSchema = null; + private SerializationSchema createSerializationSchema(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation) { + SerializationSchema serializationSchema = null; if (FormatType.JSON.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getSchemaString())) { - serializationSchema = new JsonRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); + serializationSchema = new JsonCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } else if (typeInformation != null && typeInformation.getArity() != 0) { - serializationSchema = new JsonRowSerializationSchema(typeInformation); + serializationSchema = new JsonCRowSerializationSchema(typeInformation, kafkaSinkTableInfo.getUpdateMode()); } else { throw new IllegalArgumentException("sinkDataType:" + FormatType.JSON.name() + " must set schemaString（JSON Schema）or TypeInformation"); } - } else if (FormatType.CSV.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getFieldDelimiter())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.CSV.name() + " must set fieldDelimiter"); } - - final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(typeInformation); + final CsvCRowSerializationSchema.Builder serSchemaBuilder = new CsvCRowSerializationSchema.Builder(typeInformation); serSchemaBuilder.setFieldDelimiter(kafkaSinkTableInfo.getFieldDelimiter().toCharArray()[0]); - serializationSchema = serSchemaBuilder.build(); + serSchemaBuilder.setUpdateMode(kafkaSinkTableInfo.getUpdateMode()); + serializationSchema = serSchemaBuilder.build(); } else if (FormatType.AVRO.name().equalsIgnoreCase(kafkaSinkTableInfo.getSinkDataType())) { - if (StringUtils.isBlank(kafkaSinkTableInfo.getSchemaString())) { throw new IllegalArgumentException("sinkDataType:" + FormatType.AVRO.name() + " must set schemaString"); } - - serializationSchema = new AvroRowSerializationSchema(kafkaSinkTableInfo.getSchemaString()); - + serializationSchema = new AvroCRowSerializationSchema(kafkaSinkTableInfo.getSchemaString(), kafkaSinkTableInfo.getUpdateMode()); } if (null == serializationSchema) { diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java new file mode 100644 index 000000000..7234216a7 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Optional; +import java.util.Properties; +import java.util.stream.IntStream; + +/** + * Date: 2020/4/1 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSink implements RetractStreamTableSink, IStreamSinkGener { + + public static final String SINK_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected String[] fieldNames; + protected TypeInformation[] fieldTypes; + + protected String[] partitionKeys; + protected String sinkOperatorName; + protected Properties properties; + protected int parallelism; + protected String topic; + protected String tableName; + + protected TableSchema schema; + protected SinkFunction kafkaProducer; + + + protected Optional> partitioner; + + protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaSinkTableInfo.getBootstrapServers()); + + for (String key : KafkaSinkTableInfo.getKafkaParamKeys()) { + props.setProperty(key, KafkaSinkTableInfo.getKafkaParam(key)); + } + return props; + } + + protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + return types; + } + + + protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] fieldTypes) { + Preconditions.checkArgument(fieldNames.length == fieldTypes.length, "fieldNames length must equals fieldTypes length !"); + + TableSchema.Builder builder = TableSchema.builder(); + IntStream.range(0, fieldTypes.length) + .forEach(i -> builder.field(fieldNames[i], fieldTypes[i])); + + return builder.build(); + } + + + @Override + public void emitDataStream(DataStream> dataStream) { + DataStream mapDataStream = dataStream + .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .returns(getRowTypeInfo()) + .setParallelism(parallelism); + + mapDataStream.addSink(kafkaProducer).name(sinkOperatorName); + } + + public CRowTypeInfo getRowTypeInfo() { + return new CRowTypeInfo(new RowTypeInfo(fieldTypes, fieldNames)); + } + + protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { + if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { + return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + } + return null; + } + + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } + + @Override + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + return this; + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java index e212d1f57..90dfe996b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkPartition.java @@ -11,13 +11,13 @@ public class CustomerFlinkPartition extends FlinkKafkaPartitioner { public CustomerFlinkPartition() { } - + @Override public void open(int parallelInstanceId, int parallelInstances) { Preconditions.checkArgument(parallelInstanceId >= 0, "Id of this subtask cannot be negative."); Preconditions.checkArgument(parallelInstances > 0, "Number of subtasks must be larger than 0."); this.parallelInstanceId = parallelInstanceId; } - + @Override public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) { Preconditions.checkArgument(partitions != null && partitions.length > 0, "Partitions of the target topic is empty."); if(key == null){ @@ -25,11 +25,11 @@ public int partition(T record, byte[] key, byte[] value, String targetTopic, int } return partitions[Math.abs(new String(key).hashCode()) % partitions.length]; } - + @Override public boolean equals(Object o) { return this == o || o instanceof CustomerFlinkPartition; } - + @Override public int hashCode() { return CustomerFlinkPartition.class.hashCode(); } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java index 498766564..cde8d1b1d 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKeyedSerializationSchema.java @@ -2,18 +2,20 @@ import com.dtstack.flink.sql.format.SerializationMetricWrapper; +import com.dtstack.flink.sql.sink.kafka.serialization.JsonCRowSerializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicLong; -public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { +public class CustomerKeyedSerializationSchema implements KeyedSerializationSchema { private static final Logger LOG = LoggerFactory.getLogger(CustomerKeyedSerializationSchema.class); @@ -30,38 +32,41 @@ public CustomerKeyedSerializationSchema(SerializationMetricWrapper serialization this.mapper = new ObjectMapper(); } - public byte[] serializeKey(Row element) { - if(partitionKeys == null || partitionKeys.length <=0){ + @Override + public byte[] serializeKey(CRow element) { + if (partitionKeys == null || partitionKeys.length <= 0) { return null; - } - SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); - if(serializationSchema instanceof JsonRowSerializationSchema){ - return serializeJsonKey((JsonRowSerializationSchema) serializationSchema, element); + } + SerializationSchema serializationSchema = serializationMetricWrapper.getSerializationSchema(); + if (serializationSchema instanceof JsonCRowSerializationSchema) { + return serializeJsonKey((JsonCRowSerializationSchema) serializationSchema, element); } return null; } - public byte[] serializeValue(Row element) { + @Override + public byte[] serializeValue(CRow element) { return this.serializationMetricWrapper.serialize(element); } - public String getTargetTopic(Row element) { + @Override + public String getTargetTopic(CRow element) { return null; } - private byte[] serializeJsonKey(JsonRowSerializationSchema jsonRowSerializationSchema, Row element) { + private byte[] serializeJsonKey(JsonCRowSerializationSchema jsonCRowSerializationSchema, CRow element) { try { - byte[] data = jsonRowSerializationSchema.serialize(element); + byte[] data = jsonCRowSerializationSchema.serialize(element); ObjectNode objectNode = mapper.readValue(data, ObjectNode.class); StringBuilder sb = new StringBuilder(); - for(String key : partitionKeys){ - if(objectNode.has(key)){ + for (String key : partitionKeys) { + if (objectNode.has(key)) { sb.append(objectNode.get(key.trim())); } } return sb.toString().getBytes(); - } catch (Exception e){ - if(COUNTER.getAndIncrement() % 1000 == 0){ + } catch (Exception e) { + if (COUNTER.getAndIncrement() % 1000 == 0) { LOG.error("serializeJsonKey error", e); } } diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java new file mode 100644 index 000000000..692e208b5 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/AvroCRowSerializationSchema.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecord; +import org.apache.avro.util.Utf8; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.TimeZone; +import java.util.stream.Collectors; + +/** + * Serialization schema that serializes CROW into Avro bytes. + * + *

Serializes objects that are represented in (nested) Flink rows. It support types that + * are compatible with Flink's Table & SQL API. + ** + * @author maqi + */ +public class AvroCRowSerializationSchema implements SerializationSchema { + + /** + * Used for time conversions from SQL types. + */ + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + + /** + * Avro record class for serialization. Might be null if record class is not available. + */ + private Class recordClazz; + + /** + * Schema string for deserialization. + */ + private String schemaString; + + /** + * Avro serialization schema. + */ + private transient Schema schema; + + /** + * Writer to serialize Avro record into a byte array. + */ + private transient DatumWriter datumWriter; + + /** + * Output stream to serialize records into byte array. + */ + private transient ByteArrayOutputStream arrayOutputStream; + + /** + * Low-level class for serialization of Avro values. + */ + private transient Encoder encoder; + + private String updateMode; + + private String retractKey = "retract"; + + /** + * Creates an Avro serialization schema for the given specific record class. + * + * @param recordClazz Avro record class used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(Class recordClazz, String updateMode) { + Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); + this.recordClazz = recordClazz; + this.schema = SpecificData.get().getSchema(recordClazz); + this.schemaString = schema.toString(); + this.datumWriter = new SpecificDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; + } + + /** + * Creates an Avro serialization schema for the given Avro schema string. + * + * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record + */ + public AvroCRowSerializationSchema(String avroSchemaString,String updateMode) { + Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); + this.recordClazz = null; + this.schemaString = avroSchemaString; + try { + this.schema = new Schema.Parser().parse(avroSchemaString); + } catch (SchemaParseException e) { + throw new IllegalArgumentException("Could not parse Avro schema string.", e); + } + this.datumWriter = new GenericDatumWriter<>(schema); + this.arrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + this.updateMode = updateMode; + } + + @Override + public byte[] serialize(CRow crow) { + try { + Row row = crow.row(); + boolean change = crow.change(); + + // convert to record + final GenericRecord record = convertRowToAvroRecord(schema, row); + + dealRetractField(change, record); + + arrayOutputStream.reset(); + datumWriter.write(record, encoder); + encoder.flush(); + return arrayOutputStream.toByteArray(); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize row.", e); + } + } + + protected void dealRetractField(boolean change, GenericRecord record) { + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .findFirst() + .ifPresent(field -> { + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + record.put(retractKey, convertFlinkType(field.schema(), change)); + } + }); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AvroCRowSerializationSchema that = (AvroCRowSerializationSchema) o; + return Objects.equals(recordClazz, that.recordClazz) && Objects.equals(schemaString, that.schemaString); + } + + @Override + public int hashCode() { + return Objects.hash(recordClazz, schemaString); + } + + // -------------------------------------------------------------------------------------------- + + private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { + + final List fields = schema.getFields() + .stream() + .filter(field -> !StringUtils.equalsIgnoreCase(field.name(), retractKey)) + .collect(Collectors.toList()); + + final int length = fields.size(); + final GenericRecord record = new GenericData.Record(schema); + for (int i = 0; i < length; i++) { + final Schema.Field field = fields.get(i); + record.put(i, convertFlinkType(field.schema(), row.getField(i))); + } + return record; + } + + private Object convertFlinkType(Schema schema, Object object) { + if (object == null) { + return null; + } + switch (schema.getType()) { + case RECORD: + if (object instanceof Row) { + return convertRowToAvroRecord(schema, (Row) object); + } + throw new IllegalStateException("Row expected but was: " + object.getClass()); + case ENUM: + return new GenericData.EnumSymbol(schema, object.toString()); + case ARRAY: + final Schema elementSchema = schema.getElementType(); + final Object[] array = (Object[]) object; + final GenericData.Array convertedArray = new GenericData.Array<>(array.length, schema); + for (Object element : array) { + convertedArray.add(convertFlinkType(elementSchema, element)); + } + return convertedArray; + case MAP: + final Map map = (Map) object; + final Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + convertedMap.put( + new Utf8(entry.getKey().toString()), + convertFlinkType(schema.getValueType(), entry.getValue())); + } + return convertedMap; + case UNION: + final List types = schema.getTypes(); + final int size = types.size(); + final Schema actualSchema; + if (size == 2 && types.get(0).getType() == Schema.Type.NULL) { + actualSchema = types.get(1); + } else if (size == 2 && types.get(1).getType() == Schema.Type.NULL) { + actualSchema = types.get(0); + } else if (size == 1) { + actualSchema = types.get(0); + } else { + // generic type + return object; + } + return convertFlinkType(actualSchema, object); + case FIXED: + // check for logical type + if (object instanceof BigDecimal) { + return new GenericData.Fixed( + schema, + convertFromDecimal(schema, (BigDecimal) object)); + } + return new GenericData.Fixed(schema, (byte[]) object); + case STRING: + return new Utf8(object.toString()); + case BYTES: + // check for logical type + if (object instanceof BigDecimal) { + return ByteBuffer.wrap(convertFromDecimal(schema, (BigDecimal) object)); + } + return ByteBuffer.wrap((byte[]) object); + case INT: + // check for logical types + if (object instanceof Date) { + return convertFromDate(schema, (Date) object); + } else if (object instanceof Time) { + return convertFromTime(schema, (Time) object); + } + return object; + case LONG: + // check for logical type + if (object instanceof Timestamp) { + return convertFromTimestamp(schema, (Timestamp) object); + } + return object; + case FLOAT: + case DOUBLE: + case BOOLEAN: + return object; + } + throw new RuntimeException("Unsupported Avro type:" + schema); + } + + private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType instanceof LogicalTypes.Decimal) { + final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; + // rescale to target type + final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); + // byte array must contain the two's-complement representation of the + // unscaled integer value in big-endian byte order + return decimal.unscaledValue().toByteArray(); + } else { + throw new RuntimeException("Unsupported decimal type."); + } + } + + private int convertFromDate(Schema schema, Date date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.date()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted / 86400000L); + } else { + throw new RuntimeException("Unsupported date type."); + } + } + + private int convertFromTime(Schema schema, Time date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timeMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + final long converted = time + (long) LOCAL_TZ.getOffset(time); + return (int) (converted % 86400000L); + } else { + throw new RuntimeException("Unsupported time type."); + } + } + + private long convertFromTimestamp(Schema schema, Timestamp date) { + final LogicalType logicalType = schema.getLogicalType(); + if (logicalType == LogicalTypes.timestampMillis()) { + // adopted from Apache Calcite + final long time = date.getTime(); + return time + (long) LOCAL_TZ.getOffset(time); + } else { + throw new RuntimeException("Unsupported timestamp type."); + } + } + + private void writeObject(ObjectOutputStream outputStream) throws IOException { + outputStream.writeObject(recordClazz); + outputStream.writeObject(schemaString); // support for null + outputStream.writeObject(retractKey); + outputStream.writeObject(updateMode); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { + recordClazz = (Class) inputStream.readObject(); + schemaString = (String) inputStream.readObject(); + if (recordClazz != null) { + schema = SpecificData.get().getSchema(recordClazz); + } else { + schema = new Schema.Parser().parse(schemaString); + } + retractKey = (String) inputStream.readObject(); + updateMode = (String) inputStream.readObject(); + + datumWriter = new SpecificDatumWriter<>(schema); + arrayOutputStream = new ByteArrayOutputStream(); + encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java new file mode 100644 index 000000000..4e57b6f2a --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/CsvCRowSerializationSchema.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.csv.CsvRowDeserializationSchema; +import org.apache.flink.formats.csv.CsvRowSchemaConverter; +import org.apache.flink.formats.csv.CsvRowSerializationSchema; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Objects; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * Serialization schema that serializes an object of Flink types into a CSV bytes. + * + *

Serializes the input row into a {@link ObjectNode} and + * converts it into byte[]. + * + *

Result byte[] messages can be deserialized using {@link CsvRowDeserializationSchema}. + */ +@PublicEvolving +public final class CsvCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = 2098447220136965L; + + /** Type information describing the input CSV data. */ + private final RowTypeInfo typeInfo; + + /** Runtime instance that performs the actual work. */ + private final RuntimeConverter runtimeConverter; + + /** CsvMapper used to write {@link JsonNode} into bytes. */ + private final CsvMapper csvMapper; + + /** Schema describing the input CSV data. */ + private CsvSchema csvSchema; + + /** Object writer used to write rows. It is configured by {@link CsvSchema}. */ + private ObjectWriter objectWriter; + + /** Reusable object node. */ + private transient ObjectNode root; + + private String updateMode; + + private String retractKey = "retract"; + + private CsvCRowSerializationSchema( + RowTypeInfo typeInfo, + CsvSchema csvSchema, + String updateMode) { + this.typeInfo = typeInfo; + this.runtimeConverter = createRowRuntimeConverter(typeInfo, true); + this.csvMapper = new CsvMapper(); + this.csvSchema = csvSchema; + this.updateMode = updateMode; + this.objectWriter = csvMapper.writer(csvSchema); + } + + /** + * A builder for creating a {@link CsvRowSerializationSchema}. + */ + @PublicEvolving + public static class Builder { + + private final RowTypeInfo typeInfo; + private CsvSchema csvSchema; + private String updateMode; + + /** + * Creates a {@link CsvRowSerializationSchema} expecting the given {@link TypeInformation}. + * + * @param typeInfo type information used to create schema. + */ + public Builder(TypeInformation typeInfo) { + Preconditions.checkNotNull(typeInfo, "Type information must not be null."); + + if (!(typeInfo instanceof CRowTypeInfo)) { + throw new IllegalArgumentException("Row type information expected."); + } + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + this.typeInfo = rowTypeInfo; + this.csvSchema = CsvRowSchemaConverter.convert(rowTypeInfo); + } + + public Builder setFieldDelimiter(char c) { + this.csvSchema = this.csvSchema.rebuild().setColumnSeparator(c).build(); + return this; + } + + public Builder setLineDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + throw new IllegalArgumentException( + "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + } + this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); + return this; + } + + public Builder setArrayElementDelimiter(String delimiter) { + Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); + this.csvSchema = this.csvSchema.rebuild().setArrayElementSeparator(delimiter).build(); + return this; + } + + public Builder setQuoteCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setQuoteChar(c).build(); + return this; + } + + public Builder setEscapeCharacter(char c) { + this.csvSchema = this.csvSchema.rebuild().setEscapeChar(c).build(); + return this; + } + + public Builder setNullLiteral(String s) { + this.csvSchema = this.csvSchema.rebuild().setNullValue(s).build(); + return this; + } + + public Builder setUpdateMode(String updateMode) { + this.updateMode = updateMode; + return this; + } + + public CsvCRowSerializationSchema build() { + return new CsvCRowSerializationSchema( + typeInfo, + csvSchema, + updateMode); + } + } + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (root == null) { + root = csvMapper.createObjectNode(); + } + try { + runtimeConverter.convert(csvMapper, root, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + fillRetractField(row, change); + } + + return objectWriter.writeValueAsBytes(root); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'.", t); + } + } + + protected void fillRetractField(Row row, boolean change) { + root.put(retractKey, change); + CsvSchema.Builder newBuilder = new CsvSchema.Builder(csvSchema); + + CsvSchema.Column retractColumn = new CsvSchema.Column(row.getArity(), retractKey, CsvSchema.ColumnType.BOOLEAN); + newBuilder.addColumn(retractColumn); + csvSchema = newBuilder.build(); + + this.objectWriter = csvMapper.writer(csvSchema); + + } + + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != this.getClass()) { + return false; + } + if (this == o) { + return true; + } + final CsvCRowSerializationSchema that = (CsvCRowSerializationSchema) o; + final CsvSchema otherSchema = that.csvSchema; + + return typeInfo.equals(that.typeInfo) && + csvSchema.getColumnSeparator() == otherSchema.getColumnSeparator() && + Arrays.equals(csvSchema.getLineSeparator(), otherSchema.getLineSeparator()) && + csvSchema.getArrayElementSeparator().equals(otherSchema.getArrayElementSeparator()) && + csvSchema.getQuoteChar() == otherSchema.getQuoteChar() && + csvSchema.getEscapeChar() == otherSchema.getEscapeChar() && + Arrays.equals(csvSchema.getNullValue(), otherSchema.getNullValue()); + } + + @Override + public int hashCode() { + return Objects.hash( + typeInfo, + csvSchema.getColumnSeparator(), + csvSchema.getLineSeparator(), + csvSchema.getArrayElementSeparator(), + csvSchema.getQuoteChar(), + csvSchema.getEscapeChar(), + csvSchema.getNullValue()); + } + + // -------------------------------------------------------------------------------------------- + + private interface RuntimeConverter extends Serializable { + JsonNode convert(CsvMapper csvMapper, ContainerNode container, Object obj); + } + + private static RuntimeConverter createRowRuntimeConverter(RowTypeInfo rowTypeInfo, boolean isTopLevel) { + final TypeInformation[] fieldTypes = rowTypeInfo.getFieldTypes(); + final String[] fieldNames = rowTypeInfo.getFieldNames(); + + final RuntimeConverter[] fieldConverters = createFieldRuntimeConverters(fieldTypes); + + return assembleRowRuntimeConverter(isTopLevel, fieldNames, fieldConverters); + } + + private static RuntimeConverter[] createFieldRuntimeConverters(TypeInformation[] fieldTypes) { + final RuntimeConverter[] fieldConverters = new RuntimeConverter[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + fieldConverters[i] = createNullableRuntimeConverter(fieldTypes[i]); + } + return fieldConverters; + } + + private static RuntimeConverter assembleRowRuntimeConverter( + boolean isTopLevel, + String[] fieldNames, + RuntimeConverter[] fieldConverters) { + final int rowArity = fieldNames.length; + // top level reuses the object node container + if (isTopLevel) { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ObjectNode objectNode = (ObjectNode) container; + for (int i = 0; i < rowArity; i++) { + objectNode.set( + fieldNames[i], + fieldConverters[i].convert(csvMapper, container, row.getField(i))); + } + return objectNode; + }; + } else { + return (csvMapper, container, obj) -> { + final Row row = (Row) obj; + + validateArity(rowArity, row.getArity()); + + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (int i = 0; i < rowArity; i++) { + arrayNode.add(fieldConverters[i].convert(csvMapper, arrayNode, row.getField(i))); + } + return arrayNode; + }; + } + } + + private static RuntimeConverter createNullableRuntimeConverter(TypeInformation info) { + final RuntimeConverter valueConverter = createRuntimeConverter(info); + return (csvMapper, container, obj) -> { + if (obj == null) { + return container.nullNode(); + } + return valueConverter.convert(csvMapper, container, obj); + }; + } + + private static RuntimeConverter createRuntimeConverter(TypeInformation info) { + if (info.equals(Types.VOID)) { + return (csvMapper, container, obj) -> container.nullNode(); + } else if (info.equals(Types.STRING)) { + return (csvMapper, container, obj) -> container.textNode((String) obj); + } else if (info.equals(Types.BOOLEAN)) { + return (csvMapper, container, obj) -> container.booleanNode((Boolean) obj); + } else if (info.equals(Types.BYTE)) { + return (csvMapper, container, obj) -> container.numberNode((Byte) obj); + } else if (info.equals(Types.SHORT)) { + return (csvMapper, container, obj) -> container.numberNode((Short) obj); + } else if (info.equals(Types.INT)) { + return (csvMapper, container, obj) -> container.numberNode((Integer) obj); + } else if (info.equals(Types.LONG)) { + return (csvMapper, container, obj) -> container.numberNode((Long) obj); + } else if (info.equals(Types.FLOAT)) { + return (csvMapper, container, obj) -> container.numberNode((Float) obj); + } else if (info.equals(Types.DOUBLE)) { + return (csvMapper, container, obj) -> container.numberNode((Double) obj); + } else if (info.equals(Types.BIG_DEC)) { + return (csvMapper, container, obj) -> container.numberNode((BigDecimal) obj); + } else if (info.equals(Types.BIG_INT)) { + return (csvMapper, container, obj) -> container.numberNode((BigInteger) obj); + } else if (info.equals(Types.SQL_DATE)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIME)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info.equals(Types.SQL_TIMESTAMP)) { + return (csvMapper, container, obj) -> container.textNode(obj.toString()); + } else if (info instanceof RowTypeInfo){ + return createRowRuntimeConverter((RowTypeInfo) info, false); + } else if (info instanceof BasicArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((BasicArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof ObjectArrayTypeInfo) { + return createObjectArrayRuntimeConverter(((ObjectArrayTypeInfo) info).getComponentInfo()); + } else if (info instanceof PrimitiveArrayTypeInfo && + ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return createByteArrayRuntimeConverter(); + } + else { + throw new RuntimeException("Unsupported type information '" + info + "'."); + } + } + + private static RuntimeConverter createObjectArrayRuntimeConverter(TypeInformation elementType) { + final RuntimeConverter elementConverter = createNullableRuntimeConverter(elementType); + return (csvMapper, container, obj) -> { + final Object[] array = (Object[]) obj; + final ArrayNode arrayNode = csvMapper.createArrayNode(); + for (Object element : array) { + arrayNode.add(elementConverter.convert(csvMapper, arrayNode, element)); + } + return arrayNode; + }; + } + + private static RuntimeConverter createByteArrayRuntimeConverter() { + return (csvMapper, container, obj) -> container.binaryNode((byte[]) obj); + } + + private static void validateArity(int expected, int actual) { + if (expected != actual) { + throw new RuntimeException("Row length mismatch. " + expected + + " fields expected but was " + actual + "."); + } + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java new file mode 100644 index 000000000..bfe801d52 --- /dev/null +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.kafka.serialization; + +import com.dtstack.flink.sql.enums.EUpdateMode; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.Objects; + +/** + * + * Serialization schema that serializes an object of Flink types into a JSON bytes. + * + *

Serializes the input Flink object into a JSON string and + * converts it into byte[]. + * + */ +public class JsonCRowSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = -2885556750743978636L; + + /** Type information describing the input type. */ + private final TypeInformation typeInfo; + + /** Object mapper that is used to create output JSON objects. */ + private final ObjectMapper mapper = new ObjectMapper(); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ + private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ + private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); + + /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ + private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** Reusable object node. */ + private transient ObjectNode node; + + private String updateMode; + + private final String retractKey = "retract"; + + public JsonCRowSerializationSchema(String jsonSchema, String updateMode) { + this(JsonRowSchemaConverter.convert(jsonSchema), updateMode); + } + + /** + * Creates a JSON serialization schema for the given type information. + * + * @param typeInfo The field names of {@link Row} are used to map to JSON properties. + */ + public JsonCRowSerializationSchema(TypeInformation typeInfo, String updateMode) { + Preconditions.checkNotNull(typeInfo, "Type information"); + this.typeInfo = typeInfo; + this.updateMode = updateMode; + } + + + @Override + public byte[] serialize(CRow crow) { + Row row = crow.row(); + boolean change = crow.change(); + if (node == null) { + node = mapper.createObjectNode(); + } + + RowTypeInfo rowTypeInfo = ((CRowTypeInfo) typeInfo).rowType(); + try { + convertRow(node, rowTypeInfo, row); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + node.put(retractKey, change); + } + return mapper.writeValueAsBytes(node); + } catch (Throwable t) { + throw new RuntimeException("Could not serialize row '" + row + "'. " + + "Make sure that the schema matches the input.", t); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final JsonCRowSerializationSchema that = (JsonCRowSerializationSchema) o; + return Objects.equals(typeInfo, that.typeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(typeInfo); + } + + // -------------------------------------------------------------------------------------------- + + private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { + if (reuse == null) { + reuse = mapper.createObjectNode(); + } + final String[] fieldNames = info.getFieldNames(); + + final TypeInformation[] fieldTypes = info.getFieldTypes(); + + // validate the row + if (row.getArity() != fieldNames.length) { + throw new IllegalStateException(String.format( + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + } + + for (int i = 0; i < fieldNames.length; i++) { + final String name = fieldNames[i]; + + final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); + reuse.set(name, fieldConverted); + } + + return reuse; + } + + private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { + if (info == Types.VOID || object == null) { + return container.nullNode(); + } else if (info == Types.BOOLEAN) { + return container.booleanNode((Boolean) object); + } else if (info == Types.STRING) { + return container.textNode((String) object); + } else if (info == Types.BIG_DEC) { + // convert decimal if necessary + if (object instanceof BigDecimal) { + return container.numberNode((BigDecimal) object); + } + return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); + } else if (info == Types.BIG_INT) { + // convert integer if necessary + if (object instanceof BigInteger) { + return container.numberNode((BigInteger) object); + } + return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); + } else if (info == Types.SQL_DATE) { + return container.textNode(object.toString()); + } else if (info == Types.SQL_TIME) { + final Time time = (Time) object; + // strip milliseconds if possible + if (time.getTime() % 1000 > 0) { + return container.textNode(timeFormatWithMillis.format(time)); + } + return container.textNode(timeFormat.format(time)); + } else if (info == Types.SQL_TIMESTAMP) { + return container.textNode(timestampFormat.format((Timestamp) object)); + } else if (info instanceof RowTypeInfo) { + if (reuse != null && reuse instanceof ObjectNode) { + return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); + } else { + return convertRow(null, (RowTypeInfo) info, (Row) object); + } + } else if (info instanceof ObjectArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof BasicArrayTypeInfo) { + if (reuse != null && reuse instanceof ArrayNode) { + return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } else { + return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); + } + } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { + return container.binaryNode((byte[]) object); + } else { + // for types that were specified without JSON schema + // e.g. POJOs + try { + return mapper.valueToTree(object); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); + } + } + } + + private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { + if (reuse == null) { + reuse = mapper.createArrayNode(); + } else { + reuse.removeAll(); + } + + for (Object object : array) { + reuse.add(convert(reuse, null, info, object)); + } + return reuse; + } +} diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 8ba21b953..4ad8947a8 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -18,9 +18,10 @@ package com.dtstack.flink.sql.sink.kafka.table; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -32,12 +33,13 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSinkParser extends AbsTableParser { +public class KafkaSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); kafkaSinkTableInfo.setName(tableName); kafkaSinkTableInfo.setType(MathUtil.getString(props.get(KafkaSinkTableInfo.TYPE_KEY.toLowerCase()))); + parseFieldsInfo(fieldsInfo, kafkaSinkTableInfo); if (props.get(KafkaSinkTableInfo.SINK_DATA_TYPE) != null) { @@ -46,11 +48,14 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); @@ -59,6 +70,8 @@ public class KafkaSinkTableInfo extends TargetTableInfo { private String partitionKeys; + private String updateMode; + public void addKafkaParam(String key, String value) { kafkaParam.put(key, value); } @@ -71,7 +84,6 @@ public Set getKafkaParamKeys() { return kafkaParam.keySet(); } - public String getBootstrapServers() { return bootstrapServers; } @@ -104,14 +116,40 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } + public String getUpdateMode() { + return updateMode; + } + + public void setUpdateMode(String updateMode) { + this.updateMode = updateMode; + } + @Override public boolean check() { Preconditions.checkNotNull(getType(), "kafka of type is required"); Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); + + if (StringUtils.equalsIgnoreCase(getSinkDataType(), FormatType.AVRO.name())) { + avroParamCheck(); + } + return false; } + public void avroParamCheck() { + Preconditions.checkNotNull(schemaString, "avro type schemaInfo is required"); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + Schema schema = new Schema.Parser().parse(schemaString); + schema.getFields() + .stream() + .filter(field -> StringUtils.equalsIgnoreCase(field.name(), RETRACT_FIELD_KEY)) + .findFirst() + .orElseThrow(() -> + new NullPointerException(String.valueOf("arvo upsert mode the retract attribute must be contained in schemaInfo field "))); + } + } + public String getEnableKeyPartition() { return enableKeyPartition; } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java new file mode 100644 index 000000000..852a381e2 --- /dev/null +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.source.kafka; + +import com.dtstack.flink.sql.source.IStreamSourceGener; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; +import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.Table; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.consumer.ConsumerConfig; + +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi + */ +public abstract class AbstractKafkaSource implements IStreamSourceGener

{ + + private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + + protected Properties getKafkaProperties(KafkaSourceTableInfo kafkaSourceTableInfo) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaSourceTableInfo.getBootstrapServers()); + + if (DtStringUtil.isJson(kafkaSourceTableInfo.getOffsetReset())) { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, EKafkaOffset.NONE.name().toLowerCase()); + } else { + props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceTableInfo.getOffsetReset()); + } + + if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { + props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceTableInfo.getGroupId()); + } + + for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { + props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); + } + return props; + } + + protected String generateOperatorName(String tabName, String topicName) { + return SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", tabName); + } + + protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { + Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); + TypeInformation[] types = IntStream.range(0, fieldClasses.length) + .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .toArray(TypeInformation[]::new); + + return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + } + + protected void setStartPosition(String offset, String topicName, FlinkKafkaConsumerBase kafkaSrc) { + if (StringUtils.equalsIgnoreCase(offset, EKafkaOffset.EARLIEST.name())) { + kafkaSrc.setStartFromEarliest(); + } else if (DtStringUtil.isJson(offset)) { + Map specificStartupOffsets = buildOffsetMap(offset, topicName); + kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); + } else { + kafkaSrc.setStartFromLatest(); + } + } + + /** + * kafka offset,eg.. {"0":12312,"1":12321,"2":12312} + * @param offsetJson + * @param topicName + * @return + */ + protected Map buildOffsetMap(String offsetJson, String topicName) { + try { + Properties properties = PluginUtil.jsonStrToObject(offsetJson, Properties.class); + Map offsetMap = PluginUtil.objectToMap(properties); + Map specificStartupOffsets = offsetMap + .entrySet() + .stream() + .collect(Collectors.toMap( + (Map.Entry entry) -> new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), + (Map.Entry entry) -> Long.valueOf(entry.getValue().toString())) + ); + + return specificStartupOffsets; + } catch (Exception e) { + throw new RuntimeException("not support offsetReset type:" + offsetJson); + } + } + +} diff --git a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java similarity index 56% rename from cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java rename to kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java index 33a0233ac..476ccc8b7 100644 --- a/cassandra/cassandra-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/enums/EKafkaOffset.java @@ -16,43 +16,16 @@ * limitations under the License. */ - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; +package com.dtstack.flink.sql.source.kafka.enums; /** - * Unit test for simple App. + * Date: 2020/3/20 + * Company: www.dtstack.com + * @author maqi */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } +public enum EKafkaOffset { - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } + LATEST, + EARLIEST, + NONE } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 005bfb4c2..bef86f10d 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -19,11 +19,14 @@ package com.dtstack.flink.sql.source.kafka.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.format.FormatType; +import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; +import java.util.stream.Collectors; /** * Reason: @@ -32,33 +35,38 @@ * * @author sishu.yss */ -public class KafkaSourceParser extends AbsSourceParser { +public class KafkaSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - kafkaSourceTableInfo.setName(tableName); - kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); + kafkaSourceTableInfo.setName(tableName); + kafkaSourceTableInfo.setType(MathUtil.getString(props.get(KafkaSourceTableInfo.TYPE_KEY.toLowerCase()))); kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); - if (bootstrapServer == null || bootstrapServer.trim().equals("")) { - throw new Exception("BootstrapServers can not be empty!"); - } else { - kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); - } + kafkaSourceTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()), false)); + kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase(), EKafkaOffset.LATEST.name().toLowerCase()))); + kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - for (String key : props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafkaSourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } + + kafkaSourceTableInfo.setSchemaString(MathUtil.getString(props.get(KafkaSourceTableInfo.SCHEMA_STRING_KEY.toLowerCase()))); + kafkaSourceTableInfo.setFieldDelimiter(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.CSV_FIELD_DELIMITER_KEY.toLowerCase(), "|"))); + kafkaSourceTableInfo.setSourceDataType(MathUtil.getString(props.getOrDefault(KafkaSourceTableInfo.SOURCE_DATA_TYPE_KEY.toLowerCase(), FormatType.DT_NEST.name()))); + + Map kafkaParams = props.keySet().stream() + .filter(key -> !key.isEmpty() && key.startsWith("kafka.")) + .collect(Collectors.toMap( + key -> key.substring(6), key -> props.get(key).toString()) + ); + + kafkaSourceTableInfo.addKafkaParam(kafkaParams); kafkaSourceTableInfo.check(); + return kafkaSourceTableInfo; } } diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 4f4851335..c27eee376 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -17,11 +17,10 @@ */ - package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.format.FormatType; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; import java.util.HashMap; @@ -35,136 +34,126 @@ * @author sishu.yss */ -public class KafkaSourceTableInfo extends SourceTableInfo { +public class KafkaSourceTableInfo extends AbstractSourceTableInfo { + + public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; + public static final String TOPIC_KEY = "topic"; - public static final String TOPIC_KEY = "topic"; + public static final String TYPE_KEY = "type"; - public static final String TYPE_KEY = "type"; + public static final String GROUPID_KEY = "groupId"; - public static final String GROUPID_KEY = "groupId"; + public static final String OFFSETRESET_KEY = "offsetReset"; - public static final String OFFSETRESET_KEY = "offsetReset"; + public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; + public static final String SCHEMA_STRING_KEY = "schemaInfo"; - private String bootstrapServers; + public static final String CSV_FIELD_DELIMITER_KEY = "fieldDelimiter"; - private String topic; + public static final String SOURCE_DATA_TYPE_KEY = "sourceDataType"; - private String groupId; + private String bootstrapServers; - //latest, earliest - private String offsetReset = "latest"; + private String topic; - private String offset; + private String groupId; - private Boolean topicIsPattern = false; + private String offsetReset; - private String sourceDataType = FormatType.DT_NEST.name(); + private Boolean topicIsPattern = false; - private String schemaString; + private String sourceDataType; - private String fieldDelimiter; + private String schemaString; - public String getBootstrapServers() { - return bootstrapServers; - } + private String fieldDelimiter; - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } + public Map kafkaParam = new HashMap<>(); - public String getTopic() { - return topic; - } - public void setTopic(String topic) { - this.topic = topic; - } + public String getBootstrapServers() { + return bootstrapServers; + } - public String getGroupId() { - return groupId; - } + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } - public void setGroupId(String groupId) { - this.groupId = groupId; - } + public String getTopic() { + return topic; + } - public String getOffsetReset() { - return offsetReset; - } + public void setTopic(String topic) { + this.topic = topic; + } - public void setOffsetReset(String offsetReset) { - if(offsetReset == null){ - return; - } - this.offsetReset = offsetReset; - } + public String getGroupId() { + return groupId; + } - public String getOffset() { - return offset; - } + public void setGroupId(String groupId) { + this.groupId = groupId; + } - public void setOffset(String offset) { - if (offsetReset == null) { - return; - } - this.offset = offset; - } + public String getOffsetReset() { + return offsetReset; + } - public Boolean getTopicIsPattern() { - return topicIsPattern; - } + public void setOffsetReset(String offsetReset) { + this.offsetReset = offsetReset; + } - public void setTopicIsPattern(Boolean topicIsPattern) { - this.topicIsPattern = topicIsPattern; - } + public Boolean getTopicIsPattern() { + return topicIsPattern; + } - public Map kafkaParam = new HashMap<>(); + public void setTopicIsPattern(Boolean topicIsPattern) { + this.topicIsPattern = topicIsPattern; + } - public void addKafkaParam(String key, String value) { - kafkaParam.put(key, value); - } + public void addKafkaParam(Map kafkaParam) { + kafkaParam.putAll(kafkaParam); + } - public String getKafkaParam(String key) { - return kafkaParam.get(key); - } + public String getKafkaParam(String key) { + return kafkaParam.get(key); + } - public Set getKafkaParamKeys() { - return kafkaParam.keySet(); - } + public Set getKafkaParamKeys() { + return kafkaParam.keySet(); + } - public String getSourceDataType() { - return sourceDataType; - } + public String getSourceDataType() { + return sourceDataType; + } - public void setSourceDataType(String sourceDataType) { - this.sourceDataType = sourceDataType; - } + public void setSourceDataType(String sourceDataType) { + this.sourceDataType = sourceDataType; + } - public String getSchemaString() { - return schemaString; - } + public String getSchemaString() { + return schemaString; + } - public void setSchemaString(String schemaString) { - this.schemaString = schemaString; - } + public void setSchemaString(String schemaString) { + this.schemaString = schemaString; + } - public String getFieldDelimiter() { - return fieldDelimiter; - } + public String getFieldDelimiter() { + return fieldDelimiter; + } - public void setFieldDelimiter(String fieldDelimiter) { - this.fieldDelimiter = fieldDelimiter; - } + public void setFieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java index 71a9cc386..1cbbeafd9 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer extends FlinkKafkaProducer { +public class KafkaProducer extends FlinkKafkaProducer { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class); @@ -45,7 +45,7 @@ public class KafkaProducer extends FlinkKafkaProducer { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { + public KafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] parititonKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, parititonKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java index f3a2f40f5..6bf9014df 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducerFactory.java @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import java.util.Optional; import java.util.Properties; @@ -36,7 +36,7 @@ public class KafkaProducerFactory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d0d477f4e..632bb720e 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -44,104 +29,23 @@ * @create: 2019-11-05 11:45 * @description: **/ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer flinkKafkaProducer; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafkaSinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); - for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); + this.tableName = kafkaSinkTableInfo.getName(); + this.topic = kafkaSinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); this.fieldNames = kafkaSinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; - for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafkaSinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafkaSinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducerFactory().createKafkaProducer(kafkaSinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(flinkKafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } } diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index a445a6529..394ea86ee 100644 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -18,24 +18,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -43,75 +36,24 @@ * @create: 2019-11-05 10:55 * @description: **/ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - FlinkKafkaConsumer kafkaSrc = (FlinkKafkaConsumer) new KafkaConsumerFactory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java index c815e134a..bee1865dd 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer09 extends FlinkKafkaProducer09 { +public class KafkaProducer09 extends FlinkKafkaProducer09 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer09.class); @@ -45,7 +46,7 @@ public class KafkaProducer09 extends FlinkKafkaProducer09 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner,String[] partitionKeys) { + public KafkaProducer09(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.orElse(null)); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java index 7fb3909ee..ee3423b07 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer09Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer09Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner,String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer09(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 52caa7bd2..d22be3d59 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -46,109 +31,22 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected FlinkKafkaProducer09 kafkaProducer09; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - private String[] partitionKeys; - - protected int parallelism; - - - +public class KafkaSink extends AbstractKafkaSink{ @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka09SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); - for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); + this.tableName = kafka09SinkTableInfo.getName(); + this.topic = kafka09SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); this.fieldNames = kafka09SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka09SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer09Factory() - .createKafkaProducer(kafka09SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka09SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka09SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer09Factory().createKafkaProducer(kafka09SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer09) - .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 21ed7c3e3..9f8917761 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -17,17 +17,13 @@ */ - package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; @@ -36,7 +32,6 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -47,74 +42,23 @@ * @author xuchao */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") +public class KafkaSource extends AbstractKafkaSource { @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafkaSourceTableInfo.getTopic(); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafkaSourceTableInfo.) + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for(int i = 0; i< kafkaSourceTableInfo.getFieldClasses().length; i++){ - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - FlinkKafkaConsumer09 kafkaSrc = (FlinkKafkaConsumer09) new KafkaConsumer09Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())){ - kafkaSrc.setStartFromEarliest(); - }else if(DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for(Map.Entry entry:offsetMap.entrySet()){ - specificStartupOffsets.put(new KafkaTopicPartition(topicName,Integer.valueOf(entry.getKey())),Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - }else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } return tableEnv.fromDataStream(kafkaSource, fields); } } \ No newline at end of file diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java index 3cdc45dec..3936575ef 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ * * @author maqi */ -public class KafkaProducer010 extends FlinkKafkaProducer010 { +public class KafkaProducer010 extends FlinkKafkaProducer010 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer010.class); @@ -45,7 +45,7 @@ public class KafkaProducer010 extends FlinkKafkaProducer010 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer010(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner.get()); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java index c44a9fe86..e0e023586 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer010Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer010Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer010(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 42f1045bf..eea78e121 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,21 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -46,105 +33,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected Properties properties; - - protected int parallelism; - - protected KafkaSinkTableInfo kafka10SinkTableInfo; - - /** The schema of the table. */ - private TableSchema schema; - - private String[] partitionKeys; - +public class KafkaSink extends AbstractKafkaSink { @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { - this.kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka10SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); - - for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); - } + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; + Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); + this.tableName = kafka10SinkTableInfo.getName(); + this.topic = kafka10SinkTableInfo.getTopic(); + this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); this.fieldNames = kafka10SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka10SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - - RichSinkFunction kafkaProducer010 = new KafkaProducer010Factory().createKafkaProducer(kafka10SinkTableInfo, getOutputType().getTypeAt(1), properties, - Optional.of(new CustomerFlinkPartition<>()), partitionKeys); - - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer010).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 55cffbf1e..23989ab7e 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,17 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,74 +40,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ +public class KafkaSource extends AbstractKafkaSource { - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - - FlinkKafkaConsumer010 kafkaSrc = (FlinkKafkaConsumer010) new KafkaConsumer010Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } - - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } \ No newline at end of file diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java index 7880dd419..429d21a79 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.types.Row; +import org.apache.flink.table.runtime.types.CRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ * * @author maqi */ -public class KafkaProducer011 extends FlinkKafkaProducer011 { +public class KafkaProducer011 extends FlinkKafkaProducer011 { private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer011.class); @@ -46,7 +46,7 @@ public class KafkaProducer011 extends FlinkKafkaProducer011 { private SerializationMetricWrapper serializationMetricWrapper; - public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { + public KafkaProducer011(String topicId, SerializationSchema serializationSchema, Properties producerConfig, Optional> customPartitioner, String[] partitionKeys) { super(topicId, new CustomerKeyedSerializationSchema((SerializationMetricWrapper)serializationSchema, partitionKeys), producerConfig, customPartitioner); this.serializationMetricWrapper = (SerializationMetricWrapper) serializationSchema; } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java index e2272b16e..0cb11da82 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaProducer011Factory.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; +import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import java.util.Optional; @@ -36,7 +37,7 @@ public class KafkaProducer011Factory extends AbstractKafkaProducerFactory { @Override - public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, + public RichSinkFunction createKafkaProducer(KafkaSinkTableInfo kafkaSinkTableInfo, TypeInformation typeInformation, Properties properties, Optional> partitioner, String[] partitionKeys) { return new KafkaProducer011(kafkaSinkTableInfo.getTopic(), createSerializationMetricWrapper(kafkaSinkTableInfo, typeInformation), properties, partitioner, partitionKeys); } diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index c5da782d0..ea45280c7 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -18,23 +18,8 @@ package com.dtstack.flink.sql.sink.kafka; -import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import java.util.Optional; import java.util.Properties; @@ -49,106 +34,22 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - protected FlinkKafkaProducer011 kafkaProducer011; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - private String[] partitionKeys; - - +public class KafkaSink extends AbstractKafkaSink { @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { + public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka11SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); - for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); - } + Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); + this.tableName = kafka11SinkTableInfo.getName(); + this.topic = kafka11SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); this.fieldNames = kafka11SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; - for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka11SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i) new KafkaProducer011Factory() - .createKafkaProducer(kafka11SinkTableInfo, getOutputType().getTypeAt(1), properties, partitioner, partitionKeys); + this.fieldTypes = getTypeInformations(kafka11SinkTableInfo); + this.schema = buildTableSchema(fieldNames, fieldTypes); + this.parallelism = kafka11SinkTableInfo.getParallelism(); + this.sinkOperatorName = SINK_OPERATOR_NAME_TPL.replace("${topic}", topic).replace("${table}", tableName); + this.kafkaProducer = new KafkaProducer011Factory().createKafkaProducer(kafka11SinkTableInfo, getRowTypeInfo(), kafkaProperties, partitioner, partitionKeys); return this; } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - DataStream mapDataStream = dataStream.filter((Tuple2 record) -> record.f0) - .map((Tuple2 record) -> record.f1) - .returns(getOutputType().getTypeAt(1)) - .setParallelism(parallelism); - - mapDataStream.addSink(kafkaProducer011).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - private String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo){ - if(StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())){ - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); - } - return null; - } - } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java index 3f804fc6c..c20f0678b 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaConsumer011Factory.java @@ -31,8 +31,8 @@ /** * company: www.dtstack.com - * author: toutian * create: 2019/12/24 + * @author: toutian */ public class KafkaConsumer011Factory extends AbstractKafkaConsumerFactory { diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 1a4eed08a..f58d59d05 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -19,24 +19,16 @@ package com.dtstack.flink.sql.source.kafka; -import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.types.Row; - -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -47,76 +39,24 @@ * @author sishu.yss */ -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); - - Properties props = new Properties(); - for (String key : kafkaSourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafkaSourceTableInfo.getKafkaParam(key)); - } - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())){ - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())){ - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - // only required for Kafka 0.8 - //TODO props.setProperty("zookeeper.connect", kafka09SourceTableInfo.) - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } +public class KafkaSource extends AbstractKafkaSource { - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); + @Override + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; + String topicName = kafkaSourceTableInfo.getTopic(); - FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, props); + Properties kafkaProperties = getKafkaProperties(kafkaSourceTableInfo); + TypeInformation typeInformation = getRowTypeInformation(kafkaSourceTableInfo); + FlinkKafkaConsumer011 kafkaSrc = (FlinkKafkaConsumer011) new KafkaConsumer011Factory().createKafkaTableSource(kafkaSourceTableInfo, typeInformation, kafkaProperties); - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } + String sourceOperatorName = generateOperatorName(sourceTableInfo.getName(), topicName); + DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); + kafkaSource.setParallelism(kafkaSourceTableInfo.getParallelism()); - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); + setStartPosition(kafkaSourceTableInfo.getOffsetReset(), topicName, kafkaSrc); + String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } + return tableEnv.fromDataStream(kafkaSource, fields); + } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 63ff31c50..31f9aa13d 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -1,24 +1,23 @@ package com.dtstack.flink.sql.side.kudu; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; -import org.apache.flink.util.Preconditions; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -41,7 +40,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -public class KuduAllReqRow extends AllReqRow { +public class KuduAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 6051774809356082219L; @@ -62,7 +61,7 @@ public class KuduAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -229,9 +228,6 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { if (null != workerCount) { kuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java index ddfa73201..1241a6f37 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -12,16 +12,16 @@ import java.util.List; -public class KuduAllSideInfo extends SideInfo { +public class KuduAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 9005389633060174746L; - public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { // no use } diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e1705d796..e534a85f9 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -1,20 +1,12 @@ package com.dtstack.flink.sql.side.kudu; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; @@ -24,6 +16,13 @@ import com.stumbleupon.async.Deferred; import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -42,7 +41,7 @@ import java.util.List; import java.util.Map; -public class KuduAsyncReqRow extends AsyncReqRow { +public class KuduAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); /** @@ -65,7 +64,7 @@ public class KuduAsyncReqRow extends AsyncReqRow { private AsyncKuduScanner.AsyncKuduScannerBuilder scannerBuilder; - public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new KuduAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -95,9 +94,6 @@ private void connKuDu() throws KuduException { if (null != workerCount) { asyncKuduClientBuilder.workerCount(workerCount); } - if (null != defaultSocketReadTimeoutMs) { - asyncKuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); - } if (null != defaultOperationTimeoutMs) { asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index de75f23fb..5ace515f7 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -2,8 +2,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -12,15 +12,15 @@ import java.util.List; -public class KuduAsyncSideInfo extends SideInfo { +public class KuduAsyncSideInfo extends BaseSideInfo { - public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index c03c470a7..62d215d87 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,7 +1,7 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -9,9 +9,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSideParser extends AbsSideTableParser { +public class KuduSideParser extends AbstractSideTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -52,7 +52,7 @@ public class KuduSideParser extends AbsSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduSideTableInfo kuduSideTableInfo = new KuduSideTableInfo(); kuduSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduSideTableInfo); @@ -107,6 +107,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java index c527ec268..eb8df5ad9 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideTableInfo.java @@ -1,9 +1,9 @@ package com.dtstack.flink.sql.side.kudu.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; -public class KuduSideTableInfo extends SideTableInfo { +public class KuduSideTableInfo extends AbstractSideTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java index b750e2d4a..dfc02dee6 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/utils/KuduUtil.java @@ -191,6 +191,7 @@ public static KuduPredicate buildKuduPredicate(Schema schema, PredicateInfo info return KuduPredicate.newIsNotNullPredicate(column); case "IS NULL": return KuduPredicate.newIsNullPredicate(column); + default: } return null; diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 99d19cdef..4543b9618 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -18,13 +18,11 @@ package com.dtstack.flink.sql.sink.kudu; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; import org.apache.kudu.client.AsyncKuduClient; import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; @@ -44,7 +42,7 @@ * @author gituser * @modify xiuzhu */ -public class KuduOutputFormat extends DtRichOutputFormat { +public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; @@ -125,7 +123,7 @@ public void writeRecord(Tuple2 record) throws IOException { Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { - LOG.error("record insert failed:{}", row.toString()); + LOG.error("record insert failed ..{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 0c90ffe94..4c7c3bea8 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -3,7 +3,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kudu.table.KuduTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -39,7 +39,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; @Override - public KuduSink genStreamSink(TargetTableInfo targetTableInfo) { + public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; this.kuduMasters = kuduTableInfo.getKuduMasters(); this.tableName = kuduTableInfo.getTableName(); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index ea61bd325..20302d44f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,8 +1,8 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.math.BigDecimal; @@ -10,9 +10,9 @@ import java.sql.Timestamp; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; -public class KuduSinkParser extends AbsTableParser { +public class KuduSinkParser extends AbstractTableParser { public static final String KUDU_MASTERS = "kuduMasters"; @@ -27,7 +27,7 @@ public class KuduSinkParser extends AbsTableParser { public static final String SOCKET_READ_TIMEOUT_MS = "defaultSocketReadTimeoutMs"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { KuduTableInfo kuduTableInfo = new KuduTableInfo(); kuduTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, kuduTableInfo); @@ -86,6 +86,7 @@ public Class dbTypeConvertToJavaType(String fieldType) { return BigDecimal.class; case "binary": return byte[].class; + default: } throw new RuntimeException("不支持 " + fieldType + " 类型"); diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java index c086c7298..80e47761b 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java @@ -1,10 +1,10 @@ package com.dtstack.flink.sql.sink.kudu.table; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; -public class KuduTableInfo extends TargetTableInfo { +public class KuduTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "kudu"; diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index 14cd847b5..8f0090db5 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -21,11 +21,11 @@ import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; +import com.esotericsoftware.minlog.Log; import org.apache.commons.io.Charsets; import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; @@ -89,7 +89,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { if (StringUtils.isNotBlank(yarnConfDir)) { try { - config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); + config.setString("fs.hdfs.hadoopconf", yarnConfDir); FileSystem.initialize(config); YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); @@ -109,7 +109,7 @@ public static ClusterClient createYarnSessionClient(Options launcherOptions) { applicationId = getYarnClusterApplicationId(yarnClient); } - System.out.println("applicationId=" + applicationId.toString()); + Log.info("applicationId={}", applicationId.toString()); if (StringUtils.isEmpty(applicationId.toString())) { throw new RuntimeException("No flink session found on yarn cluster."); @@ -166,7 +166,7 @@ private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) private static ApplicationId toApplicationId(String appIdStr) { Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { + if (!"application".equals(it.next())) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); } else { try { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 3de1cdbc5..0e5089bc9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; import com.dtstack.flink.sql.option.Options; +import com.esotericsoftware.minlog.Log; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; @@ -79,7 +80,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC yarnClient.init(yarnConf); yarnClient.start(); - System.out.println("----init yarn success ----"); + Log.info("----init yarn success ----"); } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 29cc4890a..ca2d5bdd7 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -73,7 +73,6 @@ public static String submit(Options launcherOptions, JobGraph jobGraph, Configur String flinkJobId = jobGraph.getJobID().toString(); String tips = String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId); - System.out.println(tips); LOG.info(tips); return applicationId; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index f90cfd973..3373f3667 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -18,25 +18,17 @@ package com.dtstack.flink.sql.side.mongo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; -import org.apache.flink.types.Row; -import org.apache.flink.util.Collector; - -import com.dtstack.flink.sql.side.AllReqRow; +import com.dtstack.flink.sql.side.BaseAllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; @@ -44,13 +36,19 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -62,7 +60,7 @@ * * @author xuqianjin */ -public class MongoAllReqRow extends AllReqRow { +public class MongoAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = -675332795591842778L; @@ -78,7 +76,7 @@ public class MongoAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -181,34 +179,13 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String userName, String password, String database, String tableName) { + private MongoCollection getConn(String address, String database, String tableName) { MongoCollection dbCollection; - try { - MongoCredential credential; - String[] servers = StringUtils.split(address, ","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - dbCollection = db.getCollection(tableName, Document.class); - return dbCollection; - } catch (Exception e) { - throw new RuntimeException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); + dbCollection = db.getCollection(tableName, Document.class); + return dbCollection; + } private void loadData(Map>> tmpCache) throws SQLException { @@ -218,8 +195,7 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), - tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index b2d5c2686..4a33f1a69 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; @@ -36,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAllSideInfo extends SideInfo{ +public class MongoAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -1696860430075523841L; - public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 42749e935..d85a9990d 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -27,33 +27,35 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.AsyncReqRow; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; import com.dtstack.flink.sql.side.mongo.utils.MongoUtil; import com.google.common.collect.Lists; import com.mongodb.BasicDBObject; import com.mongodb.Block; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.ConnectionString; import com.mongodb.async.SingleResultCallback; import com.mongodb.async.client.MongoClient; -import com.mongodb.async.client.MongoClientSettings; +import com.mongodb.MongoClientSettings; import com.mongodb.async.client.MongoClients; import com.mongodb.async.client.MongoCollection; import com.mongodb.async.client.MongoDatabase; -import com.mongodb.connection.ClusterSettings; -import com.mongodb.connection.ConnectionPoolSettings; -import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import com.google.common.collect.Lists; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; +import org.apache.flink.types.Row; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -66,68 +68,43 @@ * * @author xuqianjin */ -public class MongoAsyncReqRow extends AsyncReqRow { +public class MongoAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -1183158242862673706L; private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); - private final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = 20; - private transient MongoClient mongoClient; private MongoDatabase db; - private MongoSideTableInfo MongoSideTableInfo; + private MongoSideTableInfo mongoSideTableInfo; - public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MongoAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - MongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); - connMongoDB(); + mongoSideTableInfo = (MongoSideTableInfo) sideInfo.getSideTableInfo(); + connMongoDb(); } - public void connMongoDB() throws Exception { - MongoCredential mongoCredential; - String[] servers = StringUtils.split(MongoSideTableInfo.getAddress(), ","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - ClusterSettings clusterSettings = ClusterSettings.builder().hosts(lists).build(); - ConnectionPoolSettings connectionPoolSettings = ConnectionPoolSettings.builder() - .maxSize(DEFAULT_MAX_DB_CONN_POOL_SIZE) + public void connMongoDb() throws Exception { + String address = mongoSideTableInfo.getAddress(); + ConnectionString connectionString = new ConnectionString(address); + + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(connectionString) .build(); - if (!StringUtils.isEmpty(MongoSideTableInfo.getUserName()) || !StringUtils.isEmpty(MongoSideTableInfo.getPassword())) { - mongoCredential = MongoCredential.createCredential(MongoSideTableInfo.getUserName(), MongoSideTableInfo.getDatabase(), - MongoSideTableInfo.getPassword().toCharArray()); - MongoClientSettings settings = MongoClientSettings.builder().credential(mongoCredential) - .clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } else { - MongoClientSettings settings = MongoClientSettings.builder().clusterSettings(clusterSettings) - .connectionPoolSettings(connectionPoolSettings) - .build(); - mongoClient = MongoClients.create(settings); - } - db = mongoClient.getDatabase(MongoSideTableInfo.getDatabase()); + mongoClient = MongoClients.create(settings); + db = mongoClient.getDatabase(mongoSideTableInfo.getDatabase()); } @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); - BasicDBObject basicDBObject = new BasicDBObject(); + BasicDBObject basicDbObject = new BasicDBObject(); for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = inputCopy.row().getField(conValIndex); @@ -135,14 +112,14 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce dealMissKey(inputCopy, resultFuture); return; } - basicDBObject.put(sideInfo.getEqualFieldList().get(i), equalObj); + basicDbObject.put(sideInfo.getEqualFieldList().get(i), equalObj); } try { // 填充谓词 sideInfo.getSideTableInfo().getPredicateInfoes().stream().map(info -> { BasicDBObject filterCondition = MongoUtil.buildFilterObject(info); if (null != filterCondition) { - basicDBObject.append(info.getFieldName(), filterCondition); + basicDbObject.append(info.getFieldName(), filterCondition); } return info; }).count(); @@ -150,7 +127,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce LOG.info("add predicate infoes error ", e); } - String key = buildCacheKey(basicDBObject.values()); + String key = buildCacheKey(basicDbObject.values()); if (openCache()) { CacheObj val = getFromCache(key); if (val != null) { @@ -172,7 +149,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } AtomicInteger atomicInteger = new AtomicInteger(0); - MongoCollection dbCollection = db.getCollection(MongoSideTableInfo.getTableName(), Document.class); + MongoCollection dbCollection = db.getCollection(mongoSideTableInfo.getTableName(), Document.class); List cacheContent = Lists.newArrayList(); Block printDocumentBlock = new Block() { @Override @@ -198,7 +175,7 @@ public void onResult(final Void result, final Throwable t) { } } }; - dbCollection.find(basicDBObject).forEach(printDocumentBlock, callbackWhenFinished); + dbCollection.find(basicDbObject).forEach(printDocumentBlock, callbackWhenFinished); } @Override diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index de81ed1f1..19534d4e1 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -20,16 +20,13 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import java.util.List; @@ -39,16 +36,16 @@ * * @author xuqianjin */ -public class MongoAsyncSideInfo extends SideInfo { +public class MongoAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -3694857194254465989L; - public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { } diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 5edc12419..4fe1ebee1 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -19,15 +19,13 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -35,7 +33,7 @@ * * @author xuqianjin */ -public class MongoSideParser extends AbsSideTableParser { +public class MongoSideParser extends AbstractSideTableParser { public static final String ADDRESS_KEY = "address"; @@ -48,7 +46,7 @@ public class MongoSideParser extends AbsSideTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoSideTableInfo mongoSideTableInfo = new MongoSideTableInfo(); mongoSideTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoSideTableInfo); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index a5c834469..c83de7bbc 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.mongo.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -28,7 +28,7 @@ * * @author xuqianjin */ -public class MongoSideTableInfo extends SideTableInfo { +public class MongoSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index 2f1a2092d..cbcba8f97 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -55,6 +55,7 @@ public static BasicDBObject buildFilterObject(PredicateInfo info) { return new BasicDBObject("$exists", true); case "IS NULL": return new BasicDBObject("$exists", false); + default: } return null; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 337fbad59..3f1f9f7e2 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -19,20 +19,16 @@ package com.dtstack.flink.sql.sink.mongo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; +import com.mongodb.MongoClientURI; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; -import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.types.Row; import org.bson.Document; import org.bson.types.ObjectId; import org.slf4j.Logger; @@ -48,7 +44,7 @@ * * @author xuqianjin */ -public class MongoOutputFormat extends DtRichOutputFormat { +public class MongoOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(MongoOutputFormat.class); private String address; @@ -125,30 +121,8 @@ public void close() { } private void establishConnection() { - try { - MongoCredential credential; - String[] servers = StringUtils.split(address, ","); - String host; - Integer port; - String[] hostAndPort; - List lists = new ArrayList<>(); - for (String server : servers) { - hostAndPort = StringUtils.split(server, ":"); - host = hostAndPort[0]; - port = Integer.parseInt(hostAndPort[1]); - lists.add(new ServerAddress(host, port)); - } - if (!StringUtils.isEmpty(userName) || !StringUtils.isEmpty(password)) { - credential = MongoCredential.createCredential(userName, database, password.toCharArray()); - // To connect to mongodb server - mongoClient = new MongoClient(lists, credential, new MongoClientOptions.Builder().build()); - } else { - mongoClient = new MongoClient(lists); - } - db = mongoClient.getDatabase(database); - } catch (Exception e) { - throw new IllegalArgumentException("[connMongoDB]:" + e.getMessage()); - } + mongoClient = new MongoClient(new MongoClientURI(address)); + db = mongoClient.getDatabase(database); } private MongoOutputFormat() { diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 4e28d8fd2..3f34055ec 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.mongo.table.MongoTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -54,7 +54,7 @@ public MongoSink() { } @Override - public MongoSink genStreamSink(TargetTableInfo targetTableInfo) { + public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { MongoTableInfo mongoTableInfo = (MongoTableInfo) targetTableInfo; this.address = mongoTableInfo.getAddress(); this.tableName = mongoTableInfo.getTableName(); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java index d4810fb6d..f093b70aa 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java @@ -19,13 +19,13 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; +import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY; /** * Reason: @@ -34,7 +34,7 @@ * @author xuqianjin */ -public class MongoSinkParser extends AbsTableParser { +public class MongoSinkParser extends AbstractTableParser { private static final String CURR_TYPE = "mongo"; @@ -49,7 +49,7 @@ public class MongoSinkParser extends AbsTableParser { public static final String DATABASE_KEY = "database"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { MongoTableInfo mongoTableInfo = new MongoTableInfo(); mongoTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, mongoTableInfo); diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index 02a96d6bb..c9d94dfe6 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.mongo.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; /** @@ -30,7 +30,7 @@ */ -public class MongoTableInfo extends TargetTableInfo { +public class MongoTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "mongo"; diff --git a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mongo/mongo-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index b6b7f45cb..eb294e64e 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author xuchao */ -public class MysqlAllReqRow extends RdbAllReqRow { +public class MysqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class MysqlAllReqRow extends RdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(MYSQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java index 07b06ba38..503dbf9a3 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class MysqlAllSideInfo extends RdbAllSideInfo { - public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index de673aea7..d52f1918f 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,8 +30,6 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; @@ -44,12 +42,9 @@ */ public class MysqlAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(MysqlAsyncReqRow.class); - private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -76,7 +71,7 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java index 5d6ed370d..9555897aa 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class MysqlAsyncSideInfo extends RdbAsyncSideInfo { - public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public MysqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java index 40f68e7e4..b0a38e344 100644 --- a/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java +++ b/mysql/mysql-side/mysql-side-core/src/main/java/com/dtstack/flink/sql/side/mysql/table/MysqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mysql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class MysqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 8c1146ede..0a1749f04 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; @@ -33,7 +33,7 @@ * @author xuchao */ -public class MysqlSink extends RdbSink implements IStreamSinkGener { +public class MysqlSink extends AbstractRdbSink implements IStreamSinkGener { public MysqlSink() { super(new MySQLDialect()); @@ -42,7 +42,7 @@ public MysqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java index 2247eb8cc..49105a7a8 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/table/MysqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mysql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -36,8 +36,8 @@ public class MysqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "mysql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index 33a0233ac..000000000 --- a/mysql/mysql-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest( String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 18d9ba045..3be6687ec 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; @@ -36,22 +36,22 @@ /** * side operator with cache for all(period reload) */ -public class OracleAllReqRow extends RdbAllReqRow { +public class OracleAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(OracleAllReqRow.class); private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(ORACLE_DRIVER); Map addParams = Maps.newHashMap(); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 361366929..342533681 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -19,18 +19,17 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; public class OracleAllSideInfo extends RdbAllSideInfo { - public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 20904a061..938fd870c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -30,19 +30,14 @@ import io.vertx.ext.jdbc.JDBCClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; public class OracleAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(OracleAsyncReqRow.class); - private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; - public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -68,6 +63,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index 8e801970f..658020b1c 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -20,19 +20,20 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; + import java.util.List; public class OracleAsyncSideInfo extends RdbAsyncSideInfo { - public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -55,7 +56,7 @@ public String wrapperPlaceholder(String fieldName) { String rpadFormat = "rpad(?, %d, ' ')"; if (StringUtils.contains(type.toLowerCase(), "char")) { - TableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = sideTableInfo.getFieldExtraInfoList().get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(rpadFormat, charLength); diff --git a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java index f9124bd10..897b77d30 100644 --- a/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java +++ b/oracle/oracle-side/oracle-side-core/src/main/java/com/dtstack/flink/sql/side/oracle/table/OracleSideParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.side.oracle.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -27,8 +27,8 @@ public class OracleSideParser extends RdbSideParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index a687aa012..3a320658c 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.oracle; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; @@ -41,7 +41,7 @@ public class OracleDialect implements JDBCDialect { private List fieldList; private List fieldTypeList; - private List fieldExtraInfoList; + private List fieldExtraInfoList; @Override public boolean canHandle(String url) { @@ -133,7 +133,7 @@ public String wrapperPlaceholder(String fieldName) { String type = fieldTypeList.get(pos); if (StringUtils.contains(type.toLowerCase(), DEAL_CHAR_KEY)) { - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfoList.get(pos); int charLength = fieldExtraInfo == null ? 0 : fieldExtraInfo.getLength(); if (charLength > 0) { return String.format(RPAD_FORMAT, charLength); @@ -151,7 +151,7 @@ public void setFieldTypeList(List fieldTypeList) { this.fieldTypeList = fieldTypeList; } - public void setFieldExtraInfoList(List fieldExtraInfoList) { + public void setFieldExtraInfoList(List fieldExtraInfoList) { this.fieldExtraInfoList = fieldExtraInfoList; } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index 7a37aa6cd..f916b55f2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class OracleSink extends RdbSink implements IStreamSinkGener { +public class OracleSink extends AbstractRdbSink implements IStreamSinkGener { public OracleSink() { super(new OracleDialect()); @@ -42,7 +42,7 @@ public JDBCUpsertOutputFormat getOutputFormat() { ((OracleDialect) jdbcDialect).setFieldExtraInfoList(fieldExtraInfoList); JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index aff096bd3..1732c8cc2 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.oracle.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class OracleSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "oracle"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); oracleTableInfo.setType(CURR_TYPE); return oracleTableInfo; } diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java index f9e0950dd..5fbf0824b 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllReqRow.java @@ -1,9 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -15,7 +32,12 @@ import java.util.List; import java.util.Map; -public class PolardbAllReqRow extends RdbAllReqRow { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -23,18 +45,18 @@ public class PolardbAllReqRow extends RdbAllReqRow { private static final String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(POLARDB_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java index 822cbaa70..c46c9dfd6 100644 --- a/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java +++ b/polardb/polardb-side/polardb-all-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAllSideInfo.java @@ -1,15 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAllSideInfo extends RdbAllSideInfo { - public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 52e57e262..13d25b111 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -1,8 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -16,13 +33,18 @@ import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(PolardbAsyncReqRow.class); private final static String POLARDB_DRIVER = "com.mysql.cj.jdbc.Driver"; - public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PolardbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -48,6 +70,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java index bad71375f..15237956b 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncSideInfo.java @@ -1,16 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbAsyncSideInfo extends RdbAsyncSideInfo { - public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PolardbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } diff --git a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java index 348b3aed9..71aad7cbb 100644 --- a/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java +++ b/polardb/polardb-side/polardb-side-core/src/main/java/com/dtstack/flink/sql/side/polardb/table/PolardbSideParser.java @@ -1,16 +1,39 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.side.polardb.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSideParser extends RdbSideParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo mysqlTableInfo = super.getTableInfo(tableName, fieldsInfo, props); mysqlTableInfo.setType(CURR_TYPE); return mysqlTableInfo; } diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java index 44d558140..c69f31e8f 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/PolardbSink.java @@ -1,11 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.sink.polardb; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; - -public class PolardbSink extends RdbSink { +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ +public class PolardbSink extends AbstractRdbSink { public PolardbSink() { super(new PolardbDialect()); } @@ -13,7 +35,7 @@ public PolardbSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java index 68eead938..b4f02665e 100644 --- a/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java +++ b/polardb/polardb-sink/src/main/java/com/dtstack/flink/sql/sink/polardb/table/PolardbSinkParser.java @@ -1,16 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.dtstack.flink.sql.sink.polardb.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; +/** + * Date: 2019/12/20 + * Company: www.dtstack.com + * @author yinxi + */ public class PolardbSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "polardb"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo polardbTableInfo = super.getTableInfo(tableName, fieldsInfo, props); polardbTableInfo.setType(CURR_TYPE); return polardbTableInfo; } diff --git a/pom.xml b/pom.xml index 2794f9004..de963240c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ mysql hbase elasticsearch5 + elasticsearch6 mongo redis5 launcher @@ -33,7 +34,6 @@ impala db2 polardb - elasticsearch6 @@ -54,6 +54,20 @@ + + org.jacoco + jacoco-maven-plugin + 0.7.8 + + + + prepare-agent + report + + + + + pl.project13.maven git-commit-id-plugin diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java index 6d68cfdca..0ec8f96ce 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; @@ -41,7 +41,7 @@ * @author tcm */ -public class PostgresqlAllReqRow extends RdbAllReqRow { +public class PostgresqlAllReqRow extends AbstractRdbAllReqRow { private static final long serialVersionUID = 2098635140857937717L; @@ -49,18 +49,18 @@ public class PostgresqlAllReqRow extends RdbAllReqRow { private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(POSTGRESQL_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index d383ee46d..a4a52eabc 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -35,7 +35,7 @@ */ public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index bed10f206..42aa39a51 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -21,7 +21,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -48,7 +48,7 @@ public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -71,7 +71,7 @@ public void open(Configuration parameters) throws Exception { vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index 1d89f4894..f19a488bc 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -36,7 +36,7 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java index faee2c704..be0c5ea8b 100644 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.postgresql.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -37,8 +37,8 @@ public class PostgresqlSideParser extends RdbSideParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java index cf45a6bd9..bb731152c 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java @@ -22,13 +22,13 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** * @author maqi */ -public class PostgresqlSink extends RdbSink implements IStreamSinkGener { +public class PostgresqlSink extends AbstractRdbSink implements IStreamSinkGener { public PostgresqlSink() { super(new PostgresqlDialect()); } @@ -36,7 +36,7 @@ public PostgresqlSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java index 5943b5e52..e94cfc8a0 100644 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.postgresql.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -35,8 +35,8 @@ public class PostgresqlSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "postgresql"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); pgTableInfo.setType(CURR_TYPE); return pgTableInfo; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java similarity index 96% rename from rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java rename to rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index c3220968d..bb88c7787 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -25,8 +25,9 @@ import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; + +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import com.google.common.collect.Lists; @@ -57,11 +58,11 @@ * @author maqi */ -public abstract class RdbAllReqRow extends AllReqRow { +public abstract class AbstractRdbAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 2098635140857937718L; - private static final Logger LOG = LoggerFactory.getLogger(RdbAllReqRow.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractRdbAllReqRow.class); private static final int CONN_RETRY_NUM = 3; @@ -69,7 +70,7 @@ public abstract class RdbAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public RdbAllReqRow(SideInfo sideInfo) { + public AbstractRdbAllReqRow(BaseSideInfo sideInfo) { super(sideInfo); } @@ -91,6 +92,7 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); + cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 3b7444b77..143017133 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -23,14 +23,16 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -44,20 +46,21 @@ * @author maqi */ -public class RdbAllSideInfo extends SideInfo { +public class RdbAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = -5858335638589472159L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAllSideInfo.class.getSimpleName()); - public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), sideTableInfo.getPredicateInfoes()); - System.out.println("--------dimension sql query-------\n" + sqlCondition); + LOG.info("--------dimension sql query-------\n{}" + sqlCondition); } public String getAdditionalWhereClause() { diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 971285e52..9af6d02fd 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,7 +20,9 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; @@ -30,6 +32,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import com.google.common.collect.Lists; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -50,7 +53,7 @@ * @author maqi */ -public class RdbAsyncReqRow extends AsyncReqRow { +public class RdbAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; @@ -72,20 +75,27 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSQLClient; + private transient SQLClient rdbSqlClient; - public RdbAsyncReqRow(SideInfo sideInfo) { + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); init(sideInfo); } - protected void init(SideInfo sideInfo) { + protected void init(BaseSideInfo sideInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); int defaultAsyncPoolSize = Math.min(MAX_DB_CONN_POOL_SIZE_LIMIT, DEFAULT_DB_CONN_POOL_SIZE); int rdbPoolSize = rdbSideTableInfo.getAsyncPoolSize() > 0 ? rdbSideTableInfo.getAsyncPoolSize() : defaultAsyncPoolSize; rdbSideTableInfo.setAsyncPoolSize(rdbPoolSize); } + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + LOG.info("rdb dim table config info: {} ", rdbSideTableInfo.toString()); + } + @Override public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { CRow copyCrow = new CRow(input.row(), input.change()); @@ -238,8 +248,8 @@ public Row fillData(Row input, Object line) { @Override public void close() throws Exception { super.close(); - if (rdbSQLClient != null) { - rdbSQLClient.close(); + if (rdbSqlClient != null) { + rdbSqlClient.close(); } } @@ -254,8 +264,8 @@ public String buildCacheKey(JsonArray jsonArray) { return sb.toString(); } - public void setRdbSQLClient(SQLClient rdbSQLClient) { - this.rdbSQLClient = rdbSQLClient; + public void setRdbSqlClient(SQLClient rdbSqlClient) { + this.rdbSqlClient = rdbSqlClient; } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index f3830793e..4b6782494 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -23,8 +23,8 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.PredicateInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import com.google.common.collect.Lists; @@ -33,6 +33,8 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -48,16 +50,18 @@ * @author maqi */ -public class RdbAsyncSideInfo extends SideInfo { +public class RdbAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1942629132469918611L; + private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncSideInfo.class); - public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + + public RdbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; String sideTableName = joinInfo.getSideTableName(); @@ -75,7 +79,7 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), Arrays.asList(StringUtils.split(sideSelectFields, ",")), equalFieldList, sqlJoinCompareOperate, sideTableInfo.getPredicateInfoes()); - System.out.println("----------dimension sql query-----------\n" + sqlCondition); + LOG.info("----------dimension sql query-----------\n{}", sqlCondition); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java index fcca4f0ef..542258257 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java @@ -25,13 +25,19 @@ import javax.sql.DataSource; import java.beans.PropertyVetoException; import java.sql.SQLException; - +/** + * Date: 2019/9/17 + * Company: www.dtstack.com + * @author maqi + */ public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { @Override public DataSource getDataSource(JsonObject config) throws SQLException { String url = config.getString("url"); - if (url == null) throw new NullPointerException("url cannot be null"); + if (url == null) { + throw new NullPointerException("url cannot be null"); + } String driverClass = config.getString("driver_class"); String user = config.getString("user"); String password = config.getString("password"); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index b9811b0ee..79942ae11 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -19,8 +19,8 @@ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -33,10 +33,10 @@ * @author maqi */ -public class RdbSideParser extends AbsSideTableParser { +public class RdbSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbSideTableInfo rdbTableInfo = new RdbSideTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java index fdb024b1d..46a60a10b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.side.rdb.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; /** @@ -27,7 +27,7 @@ * * @author maqi */ -public class RdbSideTableInfo extends SideTableInfo { +public class RdbSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 1205c5521..49d163d4d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,7 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + default: } return obj; } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java similarity index 90% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java index 84eea68ff..ee51c22d1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/RdbSink.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/AbstractRdbSink.java @@ -20,8 +20,8 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -44,11 +44,8 @@ * * @author maqi */ -public abstract class RdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { - - protected String driverName; - - protected String dbURL; +public abstract class AbstractRdbSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String dbUrl; protected String userName; @@ -86,20 +83,20 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab public List fieldList; public List fieldTypeList; - public List fieldExtraInfoList; + public List fieldExtraInfoList; - public RdbSink(JDBCDialect jdbcDialect) { + public AbstractRdbSink(JDBCDialect jdbcDialect) { this.jdbcDialect = jdbcDialect; } @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { + public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RdbTableInfo rdbTableInfo = (RdbTableInfo) targetTableInfo; this.batchNum = rdbTableInfo.getBatchSize() == null ? batchNum : rdbTableInfo.getBatchSize(); this.batchWaitInterval = rdbTableInfo.getBatchWaitInterval() == null ? batchWaitInterval : rdbTableInfo.getBatchWaitInterval(); this.parallelism = rdbTableInfo.getParallelism() == null ? parallelism : rdbTableInfo.getParallelism(); - this.dbURL = rdbTableInfo.getUrl(); + this.dbUrl = rdbTableInfo.getUrl(); this.userName = rdbTableInfo.getUserName(); this.password = rdbTableInfo.getPassword(); this.tableName = rdbTableInfo.getTableName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java index e9a4f391f..f476bfadf 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCOptions.java @@ -29,7 +29,7 @@ public class JDBCOptions { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -37,9 +37,9 @@ public class JDBCOptions { private String schema; private JDBCDialect dialect; - private JDBCOptions(String dbURL, String tableName, String driverName, String username, + private JDBCOptions(String dbUrl, String tableName, String driverName, String username, String password, String schema, JDBCDialect dialect) { - this.dbURL = dbURL; + this.dbUrl = dbUrl; this.tableName = tableName; this.driverName = driverName; this.username = username; @@ -48,8 +48,8 @@ private JDBCOptions(String dbURL, String tableName, String driverName, String us this.dialect = dialect; } - public String getDbURL() { - return dbURL; + public String getDbUrl() { + return dbUrl; } public String getTableName() { @@ -84,7 +84,7 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JDBCOptions) { JDBCOptions options = (JDBCOptions) o; - return Objects.equals(dbURL, options.dbURL) && + return Objects.equals(dbUrl, options.dbUrl) && Objects.equals(tableName, options.tableName) && Objects.equals(driverName, options.driverName) && Objects.equals(username, options.username) && @@ -100,7 +100,7 @@ public boolean equals(Object o) { * Builder of {@link JDBCOptions}. */ public static class Builder { - private String dbURL; + private String dbUrl; private String tableName; private String driverName; private String username; @@ -152,8 +152,8 @@ public Builder setSchema(String schema) { /** * required, JDBC DB url. */ - public Builder setDBUrl(String dbURL) { - this.dbURL = dbURL; + public Builder setDbUrl(String dbUrl) { + this.dbUrl = dbUrl; return this; } @@ -163,7 +163,7 @@ public Builder setDialect(JDBCDialect dialect) { } public JDBCOptions build() { - checkNotNull(dbURL, "No dbURL supplied."); + checkNotNull(dbUrl, "No dbURL supplied."); checkNotNull(tableName, "No tableName supplied."); if (this.driverName == null) { @@ -173,7 +173,7 @@ public JDBCOptions build() { }); } - return new JDBCOptions(dbURL, tableName, driverName, username, password, schema, dialect); + return new JDBCOptions(dbUrl, tableName, driverName, username, password, schema, dialect); } } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java index 73f17d8fd..2a3ce5e90 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/AbstractJDBCOutputFormat.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; @@ -36,7 +36,7 @@ * @see Row * @see DriverManager */ -public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat { +public abstract class AbstractJDBCOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; public static final int DEFAULT_FLUSH_MAX_SIZE = 100; @@ -47,16 +47,16 @@ public abstract class AbstractJDBCOutputFormat extends DtRichOutputFormat protected final String username; protected final String password; - private final String drivername; + private final String driverName; protected final String dbURL; protected transient Connection connection; - public AbstractJDBCOutputFormat(String username, String password, String drivername, String dbURL) { + public AbstractJDBCOutputFormat(String username, String password, String driverName, String dbUrl) { this.username = username; this.password = password; - this.drivername = drivername; - this.dbURL = dbURL; + this.driverName = driverName; + this.dbURL = dbUrl; } @Override @@ -64,7 +64,7 @@ public void configure(Configuration parameters) { } protected void establishConnection() throws SQLException, ClassNotFoundException, IOException { - JDBCUtils.forName(drivername, getClass().getClassLoader()); + JDBCUtils.forName(driverName, getClass().getClassLoader()); if (username == null) { connection = DriverManager.getConnection(dbURL); } else { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index e1b2cc6f4..e8b6dc8a4 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -25,7 +25,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.dtstack.flink.sql.sink.rdb.writer.AppendOnlyWriter; import com.dtstack.flink.sql.sink.rdb.writer.JDBCWriter; -import com.dtstack.flink.sql.sink.rdb.writer.UpsertWriter; +import com.dtstack.flink.sql.sink.rdb.writer.AbstractUpsertWriter; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -44,6 +44,7 @@ /** * An upsert OutputFormat for JDBC. + * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -83,7 +84,7 @@ public JDBCUpsertOutputFormat( long flushIntervalMills, boolean allReplace, String updateMode) { - super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbURL()); + super(options.getUsername(), options.getPassword(), options.getDriverName(), options.getDbUrl()); this.schema = options.getSchema(); this.tableName = options.getTableName(); this.dialect = options.getDialect(); @@ -111,12 +112,11 @@ public void open(int taskNumber, int numTasks) throws IOException { initMetric(); if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.APPEND.name()) || keyFields == null || keyFields.length == 0) { - String insertSQL = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); - LOG.info("execute insert sql： {}", insertSQL); - System.out.println("execute insert sql :" + insertSQL); - jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes, this); + String insertSql = dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields); + LOG.info("execute insert sql： {}", insertSql); + jdbcWriter = new AppendOnlyWriter(insertSql, fieldTypes, this); } else { - jdbcWriter = UpsertWriter.create( + jdbcWriter = AbstractUpsertWriter.create( dialect, schema, tableName, fieldNames, fieldTypes, keyFields, partitionFields, getRuntimeContext().getExecutionConfig().isObjectReuseEnabled(), allReplace, this); } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java new file mode 100644 index 000000000..e69de29bb diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index 9bbff3b4d..aca7acb6c 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -17,9 +17,8 @@ */ package com.dtstack.flink.sql.sink.rdb.table; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -31,9 +30,9 @@ * * @author maqi */ -public class RdbSinkParser extends AbsTableParser { +public class RdbSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RdbTableInfo rdbTableInfo = new RdbTableInfo(); rdbTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, rdbTableInfo); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java index e490978e4..ccdf28e48 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbTableInfo.java @@ -18,12 +18,10 @@ package com.dtstack.flink.sql.sink.rdb.table; import com.dtstack.flink.sql.enums.EUpdateMode; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; - /** * Reason: * Date: 2018/11/27 @@ -31,7 +29,7 @@ * * @author maqi */ -public class RdbTableInfo extends TargetTableInfo { +public class RdbTableInfo extends AbstractTargetTableInfo { public static final String URL_KEY = "url"; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java similarity index 84% rename from rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java rename to rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index ce3a1c680..2c070b680 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/UpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; @@ -41,14 +41,15 @@ /** * Upsert writer to deal with upsert, delete message.dd + * @author maqi */ -public abstract class UpsertWriter implements JDBCWriter { +public abstract class AbstractUpsertWriter implements JDBCWriter { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(UpsertWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractUpsertWriter.class); - public static UpsertWriter create( + public static AbstractUpsertWriter create( JDBCDialect dialect, String schema, String tableName, @@ -58,7 +59,7 @@ public static UpsertWriter create( String[] partitionFields, boolean objectReuse, boolean allReplace, - DtRichOutputFormat metricOutputFormat) { + AbstractDtRichOutputFormat metricOutputFormat) { checkNotNull(keyFields); @@ -67,20 +68,18 @@ public static UpsertWriter create( int[] pkTypes = fieldTypes == null ? null : Arrays.stream(pkFields).map(f -> fieldTypes[f]).toArray(); - String deleteSQL = dialect.getDeleteStatement(schema, tableName, keyFields); - LOG.info("deleteSQL is :{}", deleteSQL); - System.out.println("deleteSQL is :" + deleteSQL); + String deleteSql = dialect.getDeleteStatement(schema, tableName, keyFields); + LOG.info("deleteSQL is :{}", deleteSql); - Optional upsertSQL = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); - LOG.info("execute UpsertStatement: {}", upsertSQL.orElse("use UsingInsertUpdateStatement")); - System.out.println("execute UpsertStatement: " + upsertSQL.orElse("use UsingInsertUpdateStatement")); + Optional upsertSql = dialect.getUpsertStatement(schema, tableName, fieldNames, keyFields, allReplace); + LOG.info("execute UpsertStatement: {}", upsertSql.orElse("use UsingInsertUpdateStatement")); - return upsertSQL.map((Function) sql -> + return upsertSql.map((Function) sql -> new UpsertWriterUsingUpsertStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, sql, metricOutputFormat)) + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, sql, metricOutputFormat)) .orElseGet(() -> new UpsertWriterUsingInsertUpdateStatement( - fieldTypes, pkFields, pkTypes, objectReuse, deleteSQL, + fieldTypes, pkFields, pkTypes, objectReuse, deleteSql, dialect.getRowExistsStatement(tableName, keyFields), dialect.getInsertIntoStatement(schema, tableName, fieldNames, partitionFields), dialect.getUpdateStatement(tableName, fieldNames, keyFields), @@ -90,19 +89,19 @@ public static UpsertWriter create( final int[] fieldTypes; final int[] pkTypes; private final int[] pkFields; - private final String deleteSQL; + private final String deleteSql; private final boolean objectReuse; private transient Map> keyToRows; private transient PreparedStatement deleteStatement; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - private UpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSQL, boolean objectReuse, DtRichOutputFormat metricOutputFormat) { + private AbstractUpsertWriter(int[] fieldTypes, int[] pkFields, int[] pkTypes, String deleteSql, boolean objectReuse, AbstractDtRichOutputFormat metricOutputFormat) { this.fieldTypes = fieldTypes; this.pkFields = pkFields; this.pkTypes = pkTypes; - this.deleteSQL = deleteSQL; + this.deleteSql = deleteSql; this.objectReuse = objectReuse; this.metricOutputFormat = metricOutputFormat; } @@ -115,7 +114,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.deleteStatement = connection.prepareStatement(deleteSQL); + this.deleteStatement = connection.prepareStatement(deleteSql); } @Override @@ -170,7 +169,6 @@ public void executeUpdate(Connection connection) throws SQLException { } connection.commit(); } catch (Exception e) { - System.out.println(e.getCause()); // deal pg error: current transaction is aborted, commands ignored until end of transaction block connection.rollback(); connection.commit(); @@ -207,10 +205,10 @@ private Row getPrimaryKey(Row row) { // ---------------------------------------------------------------------------------------- - private static final class UpsertWriterUsingUpsertStatement extends UpsertWriter { + private static final class UpsertWriterUsingUpsertStatement extends AbstractUpsertWriter { private static final long serialVersionUID = 1L; - private final String upsertSQL; + private final String upsertSql; private transient PreparedStatement upsertStatement; @@ -219,11 +217,11 @@ private UpsertWriterUsingUpsertStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSQL, - String upsertSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.upsertSQL = upsertSQL; + String deleteSql, + String upsertSql, + AbstractDtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.upsertSql = upsertSql; } @Override @@ -234,7 +232,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - upsertStatement = connection.prepareStatement(upsertSQL); + upsertStatement = connection.prepareStatement(upsertSql); } @Override @@ -264,12 +262,12 @@ public void close() throws SQLException { } } - private static final class UpsertWriterUsingInsertUpdateStatement extends UpsertWriter { + private static final class UpsertWriterUsingInsertUpdateStatement extends AbstractUpsertWriter { private static final long serialVersionUID = 1L; - private final String existSQL; - private final String insertSQL; - private final String updateSQL; + private final String existSql; + private final String insertSql; + private final String updateSql; private transient PreparedStatement existStatement; private transient PreparedStatement insertStatement; @@ -280,15 +278,15 @@ private UpsertWriterUsingInsertUpdateStatement( int[] pkFields, int[] pkTypes, boolean objectReuse, - String deleteSQL, - String existSQL, - String insertSQL, - String updateSQL, - DtRichOutputFormat metricOutputFormat) { - super(fieldTypes, pkFields, pkTypes, deleteSQL, objectReuse, metricOutputFormat); - this.existSQL = existSQL; - this.insertSQL = insertSQL; - this.updateSQL = updateSQL; + String deleteSql, + String existSql, + String insertSql, + String updateSql, + AbstractDtRichOutputFormat metricOutputFormat) { + super(fieldTypes, pkFields, pkTypes, deleteSql, objectReuse, metricOutputFormat); + this.existSql = existSql; + this.insertSql = insertSql; + this.updateSql = updateSql; } @Override @@ -299,9 +297,9 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { super.prepareStatement(connection); - existStatement = connection.prepareStatement(existSQL); - insertStatement = connection.prepareStatement(insertSQL); - updateStatement = connection.prepareStatement(updateSQL); + existStatement = connection.prepareStatement(existSql); + insertStatement = connection.prepareStatement(insertSql); + updateStatement = connection.prepareStatement(updateSql); } @Override diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index ca2bd4c66..3559d4376 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.rdb.writer; -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; import org.slf4j.Logger; @@ -34,6 +34,7 @@ /** * Just append record to jdbc, can not receive retract/delete message. + * @author maqi */ public class AppendOnlyWriter implements JDBCWriter { @@ -41,16 +42,16 @@ public class AppendOnlyWriter implements JDBCWriter { private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyWriter.class); - private final String insertSQL; + private final String insertSql; private final int[] fieldTypes; private transient PreparedStatement statement; private transient List rows; // only use metric - private transient DtRichOutputFormat metricOutputFormat; + private transient AbstractDtRichOutputFormat metricOutputFormat; - public AppendOnlyWriter(String insertSQL, int[] fieldTypes, DtRichOutputFormat metricOutputFormat) { - this.insertSQL = insertSQL; + public AppendOnlyWriter(String insertSql, int[] fieldTypes, AbstractDtRichOutputFormat metricOutputFormat) { + this.insertSql = insertSql; this.fieldTypes = fieldTypes; this.metricOutputFormat = metricOutputFormat; } @@ -63,7 +64,7 @@ public void open(Connection connection) throws SQLException { @Override public void prepareStatement(Connection connection) throws SQLException { - this.statement = connection.prepareStatement(insertSQL); + this.statement = connection.prepareStatement(insertSql); } /** diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 0d257c5b9..5d09c8f3d 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -18,6 +18,10 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAllReqRow; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.*; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -29,22 +33,34 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisSentinelPool; import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -public class RedisAllReqRow extends AllReqRow{ +import java.util.Calendar; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicReference; +/** + * @author yanxi + */ +public class RedisAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 7578879189085344807L; @@ -62,7 +78,7 @@ public class RedisAllReqRow extends AllReqRow{ private RedisSideReqRow redisSideReqRow; - public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); this.redisSideReqRow = new RedisSideReqRow(super.sideInfo); } @@ -90,7 +106,7 @@ protected void reloadCache() { } cacheRef.set(newCache); - LOG.info("----- Redis all cacheRef reload end:{}", newCache.size()); + LOG.info("----- Redis all cacheRef reload end:{}", Calendar.getInstance()); } @Override diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index 81e7b1f4f..ec0fa0d15 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -20,25 +20,27 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; - -public class RedisAllSideInfo extends SideInfo { +/** + * @author yanxi + */ +public class RedisAllSideInfo extends BaseSideInfo { private static final long serialVersionUID = 1998703966487857613L; - public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index f2e164922..d53bbed8c 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,8 +18,18 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.BaseAsyncReqRow; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; + import com.dtstack.flink.sql.enums.ECacheContentType; -import com.dtstack.flink.sql.side.*; +import com.dtstack.flink.sql.side.CacheMissVal; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.redis.enums.RedisType; import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; @@ -29,24 +39,20 @@ import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; -import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Consumer; - -public class RedisAsyncReqRow extends AsyncReqRow { +/** + * @author yanxi + */ +public class RedisAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = -2079908694523987738L; @@ -64,7 +70,7 @@ public class RedisAsyncReqRow extends AsyncReqRow { private RedisSideReqRow redisSideReqRow; - public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); redisSideReqRow = new RedisSideReqRow(super.sideInfo); } diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java index 9bdefe8c5..c23035ec7 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncSideInfo.java @@ -20,24 +20,26 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; - -public class RedisAsyncSideInfo extends SideInfo { +/** + * @author yanxi + */ +public class RedisAsyncSideInfo extends BaseSideInfo { private static final long serialVersionUID = -4851348392924455039L; - public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public RedisAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { + public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) { String sideTableName = joinInfo.getSideTableName(); SqlNode conditionNode = joinInfo.getCondition(); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java index f8a83b868..514984f0a 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideParser.java @@ -18,16 +18,19 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.table.AbsSideTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSideTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; -public class RedisSideParser extends AbsSideTableParser { +/** + * @author yanxi + */ +public class RedisSideParser extends AbstractSideTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisSideTableInfo redisSideTableInfo = new RedisSideTableInfo(); redisSideTableInfo.setName(tableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 5007868e9..17557f916 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.ISideReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.BaseSideInfo; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -40,9 +40,9 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; - private SideInfo sideInfo; + private BaseSideInfo sideInfo; - public RedisSideReqRow(SideInfo sideInfo){ + public RedisSideReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; } @@ -72,7 +72,7 @@ public Row fillData(Row input, Object sideInput) { return row; } - public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { + public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String value) { Integer keyIndex = sideInfo.getSideFieldIndex().get(index); String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); switch (classType){ diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 88cfcb8a2..5224134c8 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.side.redis.table; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.google.common.base.Preconditions; - -public class RedisSideTableInfo extends SideTableInfo { +/** + * @author yanxi + */ +public class RedisSideTableInfo extends AbstractSideTableInfo { private static final long serialVersionUID = -1L; diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index 0bd053473..ae4fe5a4b 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -18,14 +18,13 @@ package com.dtstack.flink.sql.sink.redis; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.outputformat.DtRichOutputFormat; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; @@ -42,7 +41,10 @@ import java.util.List; import java.util.Set; -public class RedisOutputFormat extends DtRichOutputFormat { +/** + * @author yanxi + */ +public class RedisOutputFormat extends AbstractDtRichOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; @@ -140,7 +142,8 @@ private void establishConnection() { break; //集群 case 3: - jedis = new JedisCluster(addresses, timeout, timeout,10, password, poolConfig); + jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + default: } } @@ -156,10 +159,10 @@ public void writeRecord(Tuple2 record) throws IOException { return; } - HashMap map = new HashMap<>(); - for (String primaryKey : primaryKeys){ - for (int i=0; i map = new HashMap<>(8); + for (String primaryKey : primaryKeys) { + for (int i = 0; i < fieldNames.length; i++) { + if (fieldNames[i].equals(primaryKey)) { map.put(primaryKey, i); } } @@ -167,10 +170,10 @@ public void writeRecord(Tuple2 record) throws IOException { List kvList = new LinkedList<>(); for (String primaryKey : primaryKeys){ - StringBuilder primaryKV = new StringBuilder(); + StringBuilder primaryKv = new StringBuilder(); int index = map.get(primaryKey).intValue(); - primaryKV.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKV.toString()); + primaryKv.append(primaryKey).append(":").append(row.getField(index)); + kvList.add(primaryKv.toString()); } String perKey = String.join(":", kvList); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index d2e28c01f..cc49a3ba8 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.redis.table.RedisTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -33,7 +33,9 @@ import org.apache.flink.types.Row; import java.util.List; - +/** + * @author yanxi + */ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -67,7 +69,7 @@ public RedisSink(){ } @Override - public RedisSink genStreamSink(TargetTableInfo targetTableInfo) { + public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { RedisTableInfo redisTableInfo = (RedisTableInfo) targetTableInfo; this.url = redisTableInfo.getUrl(); this.database = redisTableInfo.getDatabase(); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 0c9b0a64a..e965eeecb 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -18,8 +18,8 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -27,10 +27,12 @@ import java.util.Arrays; import java.util.List; import java.util.Map; - -public class RedisSinkParser extends AbsTableParser { +/** + * @author yanxi + */ +public class RedisSinkParser extends AbstractTableParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { RedisTableInfo redisTableInfo = new RedisTableInfo(); redisTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, redisTableInfo); diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 60d6dd12f..2425f8c38 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.redis.table; -import com.dtstack.flink.sql.table.TargetTableInfo; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; - -public class RedisTableInfo extends TargetTableInfo { +/** + * @author yanxi + */ +public class RedisTableInfo extends AbstractTargetTableInfo { private static final String CURR_TYPE = "redis"; diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java index e46e0a76f..b032fdce6 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java @@ -20,7 +20,7 @@ import com.dtstack.flink.sql.format.DeserializationMetricWrapper; import com.dtstack.flink.sql.format.dtnest.DtNestRowDeserializationSchema; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.types.Row; @@ -67,7 +67,7 @@ public class CustomerSocketTextStreamFunction implements SourceFunction { ServersocketSourceTableInfo tableInfo; public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo, - Map rowAndFieldMapping, List fieldExtraInfos) { + Map rowAndFieldMapping, List fieldExtraInfos) { this.tableInfo = tableInfo; this.deserializationSchema = new DtNestRowDeserializationSchema(typeInfo, rowAndFieldMapping, fieldExtraInfos); this.deserializationMetricWrapper = new DeserializationMetricWrapper(typeInfo, deserializationSchema); @@ -94,7 +94,7 @@ public void run(SourceContext ctx) throws Exception { while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { String record = buffer.substring(0, delimPos); // truncate trailing carriage return - if (delimiter.equals("\n") && record.endsWith("\r")) { + if ("\n".equals(delimiter) && "\r".endsWith(record)) { record = record.substring(0, record.length() - 1); } try { diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java index 3ab01fb42..b4dda5f3b 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -37,7 +37,7 @@ */ public class ServersocketSource implements IStreamSourceGener
{ @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(AbstractSourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; String tableName = serversocketSourceTableInfo.getName(); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java index b52e38e59..f2d861b17 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java @@ -17,8 +17,8 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractSourceParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; import java.util.Map; @@ -30,9 +30,9 @@ * * @author maqi */ -public class ServersocketSourceParser extends AbsSourceParser { +public class ServersocketSourceParser extends AbstractSourceParser { @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); serversocketSourceTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java index 796728eb3..3123b477f 100644 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java @@ -17,7 +17,7 @@ */ package com.dtstack.flink.sql.source.serversocket.table; -import com.dtstack.flink.sql.table.SourceTableInfo; +import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; /** @@ -27,8 +27,8 @@ * * @author maqi */ -public class ServersocketSourceTableInfo extends SourceTableInfo { - //version +public class ServersocketSourceTableInfo extends AbstractSourceTableInfo { + private static final String CURR_TYPE = "serversocket"; public static final String HOSTNAME_KEY = "host"; diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index 961539fde..dbae271e4 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -20,39 +20,43 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.sql.Connection; import java.sql.DriverManager; import java.util.List; import java.util.Map; + /** - * side operator with cache for all(period reload) + * side operator with cache for all(period reload) + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi */ -public class SqlserverAllReqRow extends RdbAllReqRow { +public class SqlserverAllReqRow extends AbstractRdbAllReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAllReqRow.class); private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override - public Connection getConn(String dbURL, String userName, String password) { + public Connection getConn(String dbUrl, String userName, String password) { try { Class.forName(SQLSERVER_DRIVER); //add param useCursorFetch=true Map addParams = Maps.newHashMap(); - //addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); return DriverManager.getConnection(targetDbUrl, userName, password); } catch (Exception e) { LOG.error("", e); diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java index 2b9913e7e..a40af5697 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllSideInfo.java @@ -20,14 +20,18 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAllSideInfo extends RdbAllSideInfo { - public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index 9fc2c35cd..a63d28607 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -33,14 +33,18 @@ import java.util.List; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(SqlserverAsyncReqRow.class); private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; - public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @@ -67,6 +71,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java index 347ca9bf4..e8a487721 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncSideInfo.java @@ -20,16 +20,21 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import java.util.List; +import java.util.List; +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverAsyncSideInfo extends RdbAsyncSideInfo { - public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + public SqlserverAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } } diff --git a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java index 3631dcd8a..e06c13898 100644 --- a/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java +++ b/sqlserver/sqlserver-side/sqlserver-side-core/src/main/java/com/dtstack/flink/sql/side/sqlserver/table/SqlserverSideParser.java @@ -18,16 +18,20 @@ package com.dtstack.flink.sql.side.sqlserver.table; import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; - +/** + * Date: 2019/11/26 + * Company: www.dtstack.com + * @author maqi + */ public class SqlserverSideParser extends RdbSideParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlServerTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlServerTableInfo.setType(CURR_TYPE); return sqlServerTableInfo; } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index dbbeb820e..3c331cdd0 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -19,7 +19,7 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.RdbSink; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; /** @@ -29,7 +29,7 @@ * * @author maqi */ -public class SqlserverSink extends RdbSink implements IStreamSinkGener { +public class SqlserverSink extends AbstractRdbSink implements IStreamSinkGener { public SqlserverSink() { super(new SqlserverDialect()); @@ -38,7 +38,7 @@ public SqlserverSink() { @Override public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDBUrl(dbURL) + .setDbUrl(dbUrl) .setDialect(jdbcDialect) .setUsername(userName) .setPassword(password) diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java index a695d6c3b..5300884bc 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/table/SqlserverSinkParser.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.sink.sqlserver.table; import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; +import com.dtstack.flink.sql.table.AbstractTableInfo; import java.util.Map; @@ -33,8 +33,8 @@ public class SqlserverSinkParser extends RdbSinkParser { private static final String CURR_TYPE = "sqlserver"; @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); sqlserverTableInfo.setType(CURR_TYPE); return sqlserverTableInfo; } From c5859aae1a45bbc93d86ed56959480ee8fd4aaa3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 11 May 2020 16:04:27 +0800 Subject: [PATCH 161/378] =?UTF-8?q?hbase=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/cassandra/table/CassandraSideParser.java | 1 + docs/plugin/hbaseSink.md | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java index 62dd753b9..38b1d082a 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java @@ -99,6 +99,7 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 11 May 2020 16:05:00 +0800 Subject: [PATCH 162/378] =?UTF-8?q?hbase=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSink.md | 52 ++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index d151fc139..00c340b3d 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -41,19 +41,51 @@ hbase2.0 ## 5.样例： ``` - CREATE TABLE MyResult( - cf:info VARCHAR, - cf:name VARCHAR, - cf:channel varchar +CREATE TABLE MyTable( + name varchar, + channel varchar, + age int )WITH( - type ='hbase', - zookeeperQuorum ='xx:2181', - zookeeperParent ='/hbase', - tableName ='workerinfo01', - rowKey ='channel' + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + cf:name varchar , + cf:channel varchar + )WITH( + type ='hbase', + zookeeperQuorum ='172.16.10.104:2181,172.16.10.224:2181,172.16.10.252:2181', + zookeeperParent ='/hbase', + tableName ='myresult', + partitionedJoin ='false', + parallelism ='1', + rowKey='name,channel' ); + +insert +into + MyResult + select + channel, + name + from + MyTable a ``` ## 6.hbase数据 ### 数据内容说明 -### 数据内容示例 \ No newline at end of file +hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'-'连接 +### 数据内容示例 +hbase(main):007:0> scan 'myresult' + ROW COLUMN+CELL + roc-daishu column=cf:channel, timestamp=1589183971724, value=daishu + roc-daishu column=cf:name, timestamp=1589183971724, value=roc \ No newline at end of file From 960660be24c86a1c44edab0a92e43bbabd197b53 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 16:20:22 +0800 Subject: [PATCH 163/378] =?UTF-8?q?=E5=9B=9E=E6=BB=9A=205f30a74b=20?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=20'=E5=9B=9E=E6=BB=9A=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84test=E5=88=86=E6=94=AF=E5=8F=98=E6=9B=B4?= =?UTF-8?q?=E5=86=85=E5=AE=B9'=E6=97=A0=E6=B3=95=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java | 1 + .../main/java/com/dtstack/flink/sql/util/ParseUtils.java | 2 ++ .../dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index b4240dbf3..8854ff4ec 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -27,6 +27,7 @@ import com.google.common.base.Strings; import java.io.Serializable; +import java.util.Map; /** * Join信息 diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java index fa06a6cc3..363d8bca0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java @@ -38,6 +38,7 @@ import com.google.common.collect.HashBasedTable; +import com.google.common.collect.HashBiMap; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlJoin; @@ -48,6 +49,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.calcite.sql.SqlKind.AS; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 9af6d02fd..6005146e2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -130,7 +130,7 @@ public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exce } } - rdbSQLClient.getConnection(conn -> { + rdbSqlClient.getConnection(conn -> { if (conn.failed()) { //Treatment failures resultFuture.completeExceptionally(conn.cause()); @@ -197,9 +197,9 @@ private Object convertDataType(Object val) { } else if (val instanceof Instant) { } else if (val instanceof Timestamp) { - val = DateUtil.getStringFromTimestamp((Timestamp) val); + val = DateUtil.timestampToString((Timestamp) val); } else if (val instanceof java.util.Date) { - val = DateUtil.getStringFromDate((java.sql.Date) val); + val = DateUtil.dateToString((java.sql.Date) val); } else { val = val.toString(); } From ac7bc46d95b7c70e6a893a3643b85c5b017c8dec Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 16:59:50 +0800 Subject: [PATCH 164/378] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=9C=AC=E5=9C=B0log?= =?UTF-8?q?=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 5 ----- .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index eb470a3c0..be1020b04 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,11 +122,6 @@ junit 4.12 - - com.aiweiergou - tools-logger - ${logger.tool.version} - diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 2831eabd5..60e3c8cd9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -18,7 +18,6 @@ package com.dtstack.flink.sql.exec; -import com.aiweiergou.tool.logger.api.ChangeLogLevelProcess; import com.dtstack.flink.sql.parser.CreateFuncParser; import com.dtstack.flink.sql.parser.CreateTmpTableParser; import com.dtstack.flink.sql.parser.FlinkPlanner; @@ -354,13 +353,4 @@ public static StreamExecutionEnvironment getStreamExeEnv(Properties confProperti return env; } - - public static void setLogLevel(ParamsInfo paramsInfo){ - String logLevel = paramsInfo.getConfProp().getProperty(ConfigConstrant.LOG_LEVEL_KEY); - if(org.apache.commons.lang3.StringUtils.isBlank(logLevel)){ - return; - } - ChangeLogLevelProcess logLevelProcess = new ChangeLogLevelProcess(); - logLevelProcess.process(logLevel); - } } \ No newline at end of file From 425bdc23c5748040a176d2b168bce2eed9192a1c Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 11 May 2020 17:00:36 +0800 Subject: [PATCH 165/378] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=9C=AC=E5=9C=B0log?= =?UTF-8?q?=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/Main.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index fe925f9a6..5d7528869 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -38,7 +38,6 @@ public class Main { public static void main(String[] args) throws Exception { ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); - ExecuteProcessHelper.setLogLevel(paramsInfo); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); env.execute(paramsInfo.getName()); LOG.info("program {} execution success", paramsInfo.getName()); From 3012c91c6e542c6dd1ab11049b004f96b09e8a5d Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 11 May 2020 21:29:52 +0800 Subject: [PATCH 166/378] =?UTF-8?q?mongo=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/mongoSink.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index 8c32dc95e..bf4bbb233 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -102,7 +102,7 @@ on ``` - 数据结果： + ## 数据结果示例 向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null) From 0e21db667b3acbf630bea5ff56f5932086cda31f Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 11:48:03 +0800 Subject: [PATCH 167/378] fix merge comflict bug --- .../sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index a01e0f71a..502f23f0a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -128,7 +128,7 @@ public void onResponse(SearchResponse searchResponse) { dealCacheData(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); resultFuture.complete(rowList); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } finally { if (tmpRhlClient != null) { try { From 42e83e7eb13f8353376456203d383bd0e018a600 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 14:15:59 +0800 Subject: [PATCH 168/378] =?UTF-8?q?=E7=BC=96=E8=AF=91=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 71652042c..fbb0aa291 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -144,7 +144,7 @@ public void accept(Map values) { dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else { dealMissKey(input, resultFuture); From 811797dc454bd5504ec2caedf358bb8d8f11c818 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 15:09:49 +0800 Subject: [PATCH 169/378] =?UTF-8?q?oracle=20=E6=96=87=E6=A1=A3=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/oracleSink.md | 70 +++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/plugin/oracleSink.md b/docs/plugin/oracleSink.md index c15a6109a..93771f3d6 100644 --- a/docs/plugin/oracleSink.md +++ b/docs/plugin/oracleSink.md @@ -37,19 +37,67 @@ CREATE TABLE tableName( | tableName | oracle表名称|是|| | schema | oracle 的schema|否|当前登录用户| | parallelism | 并行度设置|否|1| +| batchSize | flush的大小|否|100| +| batchWaitInterval | flush的时间间隔，单位ms|否|1000| +| allReplace | true:新值替换旧值|否|false| +| updateMode | APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|结果表设置主键则为UPSERT| + ## 5.样例： ``` -CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR +CREATE TABLE MyTable( + name varchar, + channel varchar, + id int )WITH( - type ='oracle', - url ='jdbc:oracle:thin:@xx.xx.xx.xx:1521:orcl', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - schema = 'MQTEST', + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', parallelism ='1' - ) - ``` \ No newline at end of file + ); + + CREATE TABLE MyResult( + primarykey_id int , + name VARCHAR, + address VARCHAR + )WITH( + type ='oracle', + url ='jdbc:oracle:thin:@172.16.8.178', + userName ='system', + password ='oracle', + tableName ='YELUO_TEST_ORACLE_01', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' + ); + +insert +into + MyResult + select + id as primarykey_id, + channel as address, + name + from + MyTable a + ``` + +## 6.数据示例 +### 输入数据 +``` +{"name":"roc","id":11,"channel":"daishuyun"} +``` +### 结果数据 +``` ++---------+------+------+-----------+ +| primarykey_id | name | address | ++---------+------+------+----------+ +| 11 | roc | daishuyun | +``` \ No newline at end of file From 2051383ef4ff48ff84512bb8ca5b81147f5b4a92 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 16:19:48 +0800 Subject: [PATCH 170/378] =?UTF-8?q?rdb=20=E8=BF=9E=E6=8E=A5=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E9=85=8D=E7=BD=AE=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/AbstractSideTableInfo.java | 15 ++++++++++++++- .../flink/sql/table/AbstractSideTableParser.java | 8 ++++---- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java index 75f507684..994674546 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AbstractSideTableInfo.java @@ -55,7 +55,9 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements public static final String ASYNC_TIMEOUT_KEY = "asyncTimeout"; public static final String ASYNC_FAIL_MAX_NUM_KEY = "asyncFailMaxNum"; - + + public static final String CONNECT_RETRY_MAX_NUM_KEY = "connectRetryMaxNum"; + public static final String ASYNC_REQ_POOL_KEY = "asyncPoolSize"; private String cacheType = "none"; @@ -79,6 +81,8 @@ public abstract class AbstractSideTableInfo extends AbstractTableInfo implements private Long asyncFailMaxNum; + private Integer connectRetryMaxNum; + private List predicateInfoes = Lists.newArrayList(); public RowTypeInfo getRowTypeInfo(){ @@ -164,6 +168,7 @@ public void setAsyncFailMaxNum(Long asyncFailMaxNum) { this.asyncFailMaxNum = asyncFailMaxNum; } + public int getAsyncPoolSize() { return asyncPoolSize; } @@ -172,6 +177,14 @@ public void setAsyncPoolSize(int asyncPoolSize) { this.asyncPoolSize = asyncPoolSize; } + + public Integer getConnectRetryMaxNum(Integer defaultValue) { + return Objects.isNull(connectRetryMaxNum) ? defaultValue : connectRetryMaxNum; + } + + public void setConnectRetryMaxNum(Integer connectRetryMaxNum) { + this.connectRetryMaxNum = connectRetryMaxNum; + } @Override public String toString() { return "Cache Info{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 3aff6e2e6..276575c7e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -110,10 +110,10 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0){ - sideTableInfo.setAsyncFailMaxNum(asyncFailNum); + if(props.containsKey(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())){ + Integer connectRetryMaxNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())); + if (connectRetryMaxNum > 0){ + sideTableInfo.setConnectRetryMaxNum(connectRetryMaxNum); } } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b7667ca0e..6b639bed2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -137,7 +137,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ LOG.error("getConnection error", conn.cause()); } - if(failCounter.get() >= sideInfo.getSideTableInfo().getAsyncFailMaxNum(3L)){ + if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(3)){ dealFillDataError(input, resultFuture, conn.cause()); finishFlag.set(true); } From 95e7eaa84f55a976568b895c192248a7256ef173 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 12 May 2020 18:00:10 +0800 Subject: [PATCH 171/378] =?UTF-8?q?=E8=A7=A3=E6=9E=90columnName=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index f57956096..7e9937ad1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -182,7 +182,7 @@ private Map parseInputParam(CRow input){ if(equalObj == null){ return inputParams; } - String columnName = sideInfo.getEqualFieldList().get(conValIndex); + String columnName = sideInfo.getEqualFieldList().get(i); inputParams.put(columnName, equalObj); } return inputParams; From c80c4d38a06965690d100b0eda63e984a558ef55 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 11:30:07 +0800 Subject: [PATCH 172/378] fix impalaDialect npe --- .../com/dtstack/flink/sql/sink/impala/ImpalaDialect.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 7bb24481a..3eaffc0ad 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -19,10 +19,12 @@ package com.dtstack.flink.sql.sink.impala; import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -61,7 +63,7 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f String schemaInfo = StringUtils.isEmpty(schema) ? "" : quoteIdentifier(schema) + "."; - List partitionFieldsList = Arrays.asList(partitionFields); + List partitionFieldsList = Objects.isNull(partitionFields) ? Lists.newArrayList() : Arrays.asList(partitionFields); String columns = Arrays.stream(fieldNames) .filter(f -> !partitionFieldsList.contains(f)) @@ -72,7 +74,7 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(f -> "?") .collect(Collectors.joining(", ")); - String partitionFieldStr = Arrays.stream(partitionFields) + String partitionFieldStr = partitionFieldsList.stream() .map(field -> field.replaceAll("\"", "'")) .collect(Collectors.joining(", ")); From ae6784f040bad10971d028cd129b255eda3d2707 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 14:46:03 +0800 Subject: [PATCH 173/378] =?UTF-8?q?fix=20=E4=BB=A3=E7=A0=81merge=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 8fc1fbb02..6e7bb3408 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -787,10 +787,6 @@ private void joinFun(Object pollObj, Map localTableCache, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } -// if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ -// throw new RuntimeException("ON condition must contain all equal fields!!!"); -// } - rightScopeChild.setRowTypeInfo(sideTableInfo.getRowTypeInfo()); joinScope.addScope(leftScopeChild); From 323c6183dc9cd3fe4904c4cedaa4baba8f470040 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 16:43:52 +0800 Subject: [PATCH 174/378] =?UTF-8?q?elasticsearch=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/elasticsearch6Sink.md | 90 ++++++++++++++++++++++++++----- docs/plugin/elasticsearchSink.md | 60 +++++++++++++++++---- 2 files changed, 128 insertions(+), 22 deletions(-) diff --git a/docs/plugin/elasticsearch6Sink.md b/docs/plugin/elasticsearch6Sink.md index 0bab12cc0..0f7a04280 100644 --- a/docs/plugin/elasticsearch6Sink.md +++ b/docs/plugin/elasticsearch6Sink.md @@ -42,21 +42,85 @@ CREATE TABLE tableName( |password | 密码 | 否，authMesh='true'时为必填 || |parallelism | 并行度设置|否|1| -## 5.样例： +## 5.完整样例： ``` +CREATE TABLE MyTable( + channel varchar, + pv INT, + xctime bigint + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - aa INT, - bb INT + pv varchar, + channel varchar )WITH( type ='elasticsearch6', - address ='172.16.10.47:9500', - cluster='es_47_menghan', - esType ='type1', - index ='xc_es_test', - authMesh = 'true', - userName = 'elastic', - password = 'abc123', - id ='0,1', + address ='172.16.8.193:9200', + authMesh='true', + username='elastic', + password='abc123', + estype ='external', + cluster ='docker-cluster', + index ='myresult', + id ='1', + updateMode ='append', parallelism ='1' - ) - ``` \ No newline at end of file + ); + +CREATE TABLE sideTable( + a varchar, + b varchar, + PRIMARY KEY(a) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='elasticsearch6', + address ='172.16.8.193:9200', + estype ='external', + cluster ='docker-cluster', + index ='sidetest', + authMesh='true', + username='elastic', + password='abc123', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + partitionedJoin ='false', + parallelism ='1' + ); + +insert +into + MyResult + select + w.b as pv, + s.channel as channel + from + MyTable s + join + sideTable w + on s.pv = w.a + where + w.a = '10' + and s.channel='xc'; + ``` +## 6.结果数据 +### 输入数据 +``` +{"channel":"xc26","pv":10,"xctime":1232312} +``` +### 输出数据 +``` +http://172.16.8.193:9200/myresult/_search +{"_index":"myresult","_type":"external","_id":"8aX_DHIBn3B7OBuqFl-i","_score":1.0,"_source":{"pv":"10","channel":"xc26"}} +``` \ No newline at end of file diff --git a/docs/plugin/elasticsearchSink.md b/docs/plugin/elasticsearchSink.md index 4908f8a05..9a2a32b6f 100644 --- a/docs/plugin/elasticsearchSink.md +++ b/docs/plugin/elasticsearchSink.md @@ -40,16 +40,58 @@ CREATE TABLE tableName( ## 5.样例： ``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - aa INT, - bb INT + pv varchar, + channel varchar )WITH( type ='elasticsearch', - address ='172.16.10.47:9500', - cluster='es_47_menghan', - estype ='type1', - index ='xc_es_test', - id ='0,1', + address ='172.16.8.193:9200', + authMesh='true', + username='elastic', + password='abc123', + estype ='external', + cluster ='docker-cluster', + index ='myresult', + id ='1', + updateMode ='append', parallelism ='1' - ) - ``` \ No newline at end of file + ); + + +insert +into + MyResult + select + s.pv as pv, + s.channel as channel + from + MyTable s + + ``` + +## 6.结果示例 +### 输入数据示例 +``` +{"channel":"xc26","pv":"10","xctime":1232312} +``` +### 输出数据示例 +``` +http://172.16.8.193:9200/myresult/_search +{"_index":"myresult","_type":"external","_id":"8aX_DHIBn3B7OBuqFl-i","_score":1.0,"_source":{"pv":"10","channel":"xc26"}} +``` \ No newline at end of file From 3ba019dfd7453242eb9d97537c43f57c35ddd90b Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 13 May 2020 17:33:48 +0800 Subject: [PATCH 175/378] fix --- .../dtstack/flink/sql/table/AbstractSideTableParser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java index 276575c7e..0f0612c09 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractSideTableParser.java @@ -110,6 +110,14 @@ protected void parseCacheProp(AbstractSideTableInfo sideTableInfo, Map 0."); + } + sideTableInfo.setAsyncFailMaxNum(asyncFailMaxNum); + } + if(props.containsKey(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())){ Integer connectRetryMaxNum = MathUtil.getIntegerVal(props.get(AbstractSideTableInfo.CONNECT_RETRY_MAX_NUM_KEY.toLowerCase())); if (connectRetryMaxNum > 0){ From b282ce6988c912a737797edba02902c7cb293301 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 11:46:06 +0800 Subject: [PATCH 176/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dmongo=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5url=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/mongoSide.md | 3 +-- .../flink/sql/side/mongo/MongoAllReqRow.java | 18 +++++++++++++++--- .../flink/sql/side/mongo/MongoAsyncReqRow.java | 16 +++++++++++++--- .../sql/sink/mongo/MongoOutputFormat.java | 13 ++++++++++++- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 62d2d8698..002743256 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -62,8 +62,7 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index c1a32cb64..1857dbede 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -177,9 +177,10 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String database, String tableName) { + private MongoCollection getConn(String host, String userName, String password, String database, String tableName) { + MongoCollection dbCollection; - mongoClient = new MongoClient(new MongoClientURI(address)); + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl(host, userName, password))); db = mongoClient.getDatabase(database); dbCollection = db.getCollection(tableName, Document.class); return dbCollection; @@ -193,7 +194,8 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), + tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -251,4 +253,14 @@ private void loadData(Map>> tmpCache) throws SQ } } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 6bea9e024..04f2ad9af 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.mongo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -84,9 +85,8 @@ public void open(Configuration parameters) throws Exception { } public void connMongoDb() throws Exception { - String address = mongoSideTableInfo.getAddress(); - ConnectionString connectionString = new ConnectionString(address); - + ConnectionString connectionString = new ConnectionString(getConnectionUrl(mongoSideTableInfo.getAddress(), + mongoSideTableInfo.getUserName(), mongoSideTableInfo.getPassword())); MongoClientSettings settings = MongoClientSettings.builder() .applyConnectionString(connectionString) .build(); @@ -193,4 +193,14 @@ public void close() throws Exception { } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 37f7f5952..df8293522 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -25,6 +25,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -119,7 +120,8 @@ public void close() { } private void establishConnection() { - mongoClient = new MongoClient(new MongoClientURI(address)); + + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl())); db = mongoClient.getDatabase(database); } @@ -197,5 +199,14 @@ public MongoOutputFormat finish() { } } + private String getConnectionUrl(){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } } From abfb0839a87f874eeb796f35ad6c9564f1e9c02a Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 12:56:10 +0800 Subject: [PATCH 177/378] fix mongo url bug --- docs/mongoSide.md | 5 ++--- .../flink/sql/side/mongo/MongoAllReqRow.java | 18 +++++++++++++++--- .../flink/sql/side/mongo/MongoAsyncReqRow.java | 16 +++++++++++++--- .../sql/sink/mongo/MongoOutputFormat.java | 13 ++++++++++++- 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/mongoSide.md b/docs/mongoSide.md index 73ce9644f..002743256 100644 --- a/docs/mongoSide.md +++ b/docs/mongoSide.md @@ -30,7 +30,7 @@ |----|---| | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| + | colType | 列类型 [colType支持的类型](docs/colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| @@ -62,8 +62,7 @@ create table sideTable( PERIOD FOR SYSTEM_TIME )WITH( type ='mongo', - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]]/[?options]] - address ='mongodb://172.21.32.1:27017,172.21.32.1:27017', + address ='172.21.32.1:27017,172.21.32.1:27017', database ='test', tableName ='sidetest', cache ='LRU', diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index c1a32cb64..1857dbede 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -177,9 +177,10 @@ private String buildKey(Map val, List equalFieldList) { return sb.toString(); } - private MongoCollection getConn(String address, String database, String tableName) { + private MongoCollection getConn(String host, String userName, String password, String database, String tableName) { + MongoCollection dbCollection; - mongoClient = new MongoClient(new MongoClientURI(address)); + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl(host, userName, password))); db = mongoClient.getDatabase(database); dbCollection = db.getCollection(tableName, Document.class); return dbCollection; @@ -193,7 +194,8 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { try { - dbCollection = getConn(tableInfo.getAddress(), tableInfo.getDatabase(), tableInfo.getTableName()); + dbCollection = getConn(tableInfo.getAddress(), tableInfo.getUserName(), tableInfo.getPassword(), + tableInfo.getDatabase(), tableInfo.getTableName()); break; } catch (Exception e) { if (i == CONN_RETRY_NUM - 1) { @@ -251,4 +253,14 @@ private void loadData(Map>> tmpCache) throws SQ } } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 6bea9e024..04f2ad9af 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.mongo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -84,9 +85,8 @@ public void open(Configuration parameters) throws Exception { } public void connMongoDb() throws Exception { - String address = mongoSideTableInfo.getAddress(); - ConnectionString connectionString = new ConnectionString(address); - + ConnectionString connectionString = new ConnectionString(getConnectionUrl(mongoSideTableInfo.getAddress(), + mongoSideTableInfo.getUserName(), mongoSideTableInfo.getPassword())); MongoClientSettings settings = MongoClientSettings.builder() .applyConnectionString(connectionString) .build(); @@ -193,4 +193,14 @@ public void close() throws Exception { } } + private String getConnectionUrl(String address, String userName, String password){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } + } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index 37f7f5952..df8293522 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -25,6 +25,7 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.UpdateResult; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -119,7 +120,8 @@ public void close() { } private void establishConnection() { - mongoClient = new MongoClient(new MongoClientURI(address)); + + mongoClient = new MongoClient(new MongoClientURI(getConnectionUrl())); db = mongoClient.getDatabase(database); } @@ -197,5 +199,14 @@ public MongoOutputFormat finish() { } } + private String getConnectionUrl(){ + if(address.startsWith("mongodb://") || address.startsWith("mongodb+srv://")){ + return address; + } + if (StringUtils.isNotBlank(userName) && StringUtils.isNotBlank(password)) { + return String.format("mongodb://%s:%s@%s", userName, password, address); + } + return String.format("mongodb://%s", address); + } } From c52ffa9a164540f3afba772e0ec6666e318d71ec Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 15:18:56 +0800 Subject: [PATCH 178/378] =?UTF-8?q?hbase=20=E8=A7=A3=E6=9E=90=E5=88=97?= =?UTF-8?q?=E6=97=8F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/table/HbaseSinkParser.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index bd1b7cac3..001443daa 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -26,6 +26,7 @@ import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.MathUtil; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; @@ -98,14 +99,23 @@ public void parseFieldsInfo(String fieldsInfo, HbaseTableInfo tableInfo){ String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); Class fieldClass = dbTypeConvertToJavaType(fieldType); String[] columnFamily = StringUtils.split(fieldName.trim(), ":"); - columnFamilies.put(fieldName.trim(),columnFamily[1]); tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); tableInfo.addField(columnFamily[1]); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); tableInfo.addFieldExtraInfo(null); } - tableInfo.setColumnNameFamily(columnFamilies); + tableInfo.setColumnNameFamily(parseColumnFamily(tableInfo.getPhysicalFields())); tableInfo.finish(); } + + private Map parseColumnFamily(Map physicalFieldMap){ + Map columnFamiles = Maps.newHashMap(); + physicalFieldMap.values().forEach(x -> { + String[] columnFamily = StringUtils.split(x.trim(), ":"); + columnFamiles.put(x, columnFamily[1]); + }); + + return columnFamiles; + } } From 15c86353dcf954fb48d0f11eb05703204e18ec02 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 16:20:03 +0800 Subject: [PATCH 179/378] =?UTF-8?q?fix=20mongo=20=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E6=B2=A1=E6=9C=89=E8=A7=A3=E6=9E=90equalFiel?= =?UTF-8?q?d=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/mongo/MongoAsyncReqRow.java | 2 +- .../flink/sql/side/mongo/MongoAsyncSideInfo.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 04f2ad9af..0cf8158c7 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -96,7 +96,7 @@ public void connMongoDb() throws Exception { @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { - CRow inputCopy = new CRow(input.row(), input.change()); + CRow inputCopy = new CRow(Row.copy(input.row()), input.change()); BasicDBObject basicDbObject = new BasicDBObject(); try { basicDbObject.putAll(inputParams); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index 19534d4e1..37ec8d8c6 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -46,6 +48,16 @@ public MongoAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List sqlNodeList = Lists.newArrayList(); + ParseUtils.parseAnd(conditionNode, sqlNodeList); + + for(SqlNode sqlNode : sqlNodeList){ + dealOneEqualCon(sqlNode, sideTableName); + } } From 818c3203b5fb4d77a550c8ed9a4c1516dc2999f8 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 14 May 2020 21:51:16 +0800 Subject: [PATCH 180/378] fix mongo and async block bug --- .../flink/sql/side/mongo/utils/MongoUtil.java | 19 +++++++++++-------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 12 +++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java index cbcba8f97..1b3ce7194 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/utils/MongoUtil.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.side.PredicateInfo; import com.mongodb.BasicDBObject; import org.apache.commons.lang3.StringUtils; +import org.bson.Document; import java.util.Arrays; import java.util.stream.Collectors; @@ -32,25 +33,27 @@ */ public class MongoUtil { public static BasicDBObject buildFilterObject(PredicateInfo info) { + + String value = info.getCondition().replaceAll("'", ""); switch (info.getOperatorName()) { case "=": - return new BasicDBObject("$eq", info.getCondition()); + return new BasicDBObject("$eq", value); case ">": - return new BasicDBObject("$gt", info.getCondition()); + return new BasicDBObject("$gt", value); case ">=": - return new BasicDBObject("$gte", info.getCondition()); + return new BasicDBObject("$gte", value); case "<": - return new BasicDBObject("$lt", info.getCondition()); + return new BasicDBObject("$lt", value); case "<=": - return new BasicDBObject("$lte", info.getCondition()); + return new BasicDBObject("$lte", value); case "<>": - return new BasicDBObject("$ne", info.getCondition()); + return new BasicDBObject("$ne", value); case "IN": - Object[] values = Arrays.stream(StringUtils.split(info.getCondition(), ",")).map(String::trim) + Object[] values = Arrays.stream(StringUtils.split(value, ",")).map(String::trim) .collect(Collectors.toList()).toArray(); return new BasicDBObject("$in", values); case "NOT IN": - return new BasicDBObject("$nin", StringUtils.split(info.getCondition(), ",")); + return new BasicDBObject("$nin", StringUtils.split(value, ",")); case "IS NOT NULL": return new BasicDBObject("$exists", true); case "IS NULL": diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 6b639bed2..e9d534df2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -137,8 +137,8 @@ private void connectWithRetry(Map inputParams, CRow input, Resul if(failCounter.getAndIncrement() % 1000 == 0){ LOG.error("getConnection error", conn.cause()); } - if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(3)){ - dealFillDataError(input, resultFuture, conn.cause()); + if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ + resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } conn.result().close(); @@ -155,12 +155,18 @@ private void connectWithRetry(Map inputParams, CRow input, Resul latch.countDown(); } }); - //主线程阻塞 try { latch.wait(); } catch (InterruptedException e) { LOG.error("", e); } + if(!finishFlag.get()){ + try { + Thread.sleep(100); + } catch (Exception e){ + LOG.error("", e); + } + } } } From 9435152893df7a8ac1a99b3282119592ee12f484 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 10:10:48 +0800 Subject: [PATCH 181/378] =?UTF-8?q?kudu=E6=96=87=E6=A1=A3=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/kafkaSource.md | 5 +++ docs/plugin/kuduSink.md | 80 ++++++++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 035e39ffb..1ad247398 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -78,6 +78,11 @@ kafka.dual.commit.enabled kafka.partition.assignment.strategy kafka.socket.receive.buffer.bytes kafka.fetch.min.bytes + +###kerberos认证相关参数 +kafka.security.protocal +kafka.sasl.mechanism +kafka.sasl.kerberos.service.name ``` ## 5.样例： diff --git a/docs/plugin/kuduSink.md b/docs/plugin/kuduSink.md index 25f44ede1..b39ba1d67 100644 --- a/docs/plugin/kuduSink.md +++ b/docs/plugin/kuduSink.md @@ -46,17 +46,81 @@ kudu 1.9.0+cdh6.2.0 ## 5.样例： ``` +CREATE TABLE MyTable( + channel varchar, + name varchar, + pv varchar, + a varchar, + b varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + CREATE TABLE MyResult( - id int, - title VARCHAR, - amount decimal, - tablename1 VARCHAR + a string, + b string, + c string, + d string )WITH( type ='kudu', - kuduMasters ='localhost1,localhost2,localhost3', - tableName ='impala::default.test', - writeMode='upsert', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='myresult', + writeMode='insert', parallelism ='1' ); - ``` \ No newline at end of file +CREATE TABLE sideTable( + c string, + d string, + PRIMARY KEY(c) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='kudu', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='sidetest4', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + primaryKey ='c', + isFaultTolerant ='false' + ); + +insert +into + MyResult + select + MyTable.a, + MyTable.b, + s.c, + s.d + from + MyTable + join + sideTable s + on MyTable.a = s.c + where + MyTable.a='2' + and s.d='2' + + ``` + +## 6.数据示例 +### 输入数据 +``` +{"channel":"daishuyun","name":"roc","pv":"10","a":"2","b":"2"} +``` +### 结果数据 +``` +{"a":"2","b":"2","c":"3","d":"4"} +``` \ No newline at end of file From bbd8cf935b12d218569bc9174c86418c2ec160d0 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 11:52:56 +0800 Subject: [PATCH 182/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=85=A8=E5=B1=80?= =?UTF-8?q?=E7=9A=84=E8=BF=9E=E6=8E=A5=E7=8A=B6=E6=80=81=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index e9d534df2..7340357a9 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -82,7 +82,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient SQLClient rdbSqlClient; - private final static AtomicBoolean CONN_STATUS = new AtomicBoolean(true); + private AtomicBoolean connectionStatus = new AtomicBoolean(true); public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); @@ -105,7 +105,7 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicLong networkLogCounter = new AtomicLong(0L); - while (!CONN_STATUS.get()){//network is unhealth + while (!connectionStatus.get()){//network is unhealth if(networkLogCounter.getAndIncrement() % 1000 == 0){ LOG.info("network unhealth to block task"); } @@ -114,11 +114,11 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul Map params = formatInputParam(inputParams); rdbSqlClient.getConnection(conn -> { if(conn.failed()){ - CONN_STATUS.set(false); + connectionStatus.set(false); connectWithRetry(params, input, resultFuture, rdbSqlClient); return; } - CONN_STATUS.set(true); + connectionStatus.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), params, input, resultFuture); @@ -141,10 +141,9 @@ private void connectWithRetry(Map inputParams, CRow input, Resul resultFuture.completeExceptionally(conn.cause()); finishFlag.set(true); } - conn.result().close(); return; } - CONN_STATUS.set(true); + connectionStatus.set(true); ScheduledFuture timerFuture = registerTimer(input, resultFuture); cancelTimerWhenComplete(resultFuture, timerFuture); handleQuery(conn.result(), inputParams, input, resultFuture); From 72b4ec2de72f0130687a81d84fbe4bf42880c074 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 15 May 2020 14:24:43 +0800 Subject: [PATCH 183/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E4=BD=BF=E7=94=A8=E7=BB=B4=E8=A1=A8=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E9=A2=84=E6=9C=9F=E4=B8=8D=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/MyLocalStreamEnvironment.java | 33 ++++++++++++++----- .../flink/sql/exec/ExecuteProcessHelper.java | 12 +++++-- .../com/dtstack/flink/sql/side/JoinInfo.java | 17 ++++++++-- .../flink/sql/side/JoinNodeDealer.java | 33 ++++++++++++------- .../dtstack/flink/sql/side/SideSQLParser.java | 25 +++++++------- .../dtstack/flink/sql/side/SideSqlExec.java | 9 ++--- .../dtstack/flink/sql/util/TableUtils.java | 14 +++++--- 7 files changed, 98 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 8ac1edd41..6aec8475e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -93,34 +94,48 @@ public JobExecutionResult execute(String jobName) throws Exception { // transform the streaming program into a JobGraph StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); + return execute(streamGraph); + } + + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.setClasspaths(classpaths); + jobGraph.setAllowQueuedScheduling(true); Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); + if (!configuration.contains(RestOptions.BIND_PORT)) { + configuration.setString(RestOptions.BIND_PORT, "0"); + } + + int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + + MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(configuration) + .setNumSlotsPerTaskManager(numSlotsPerTaskManager) + .build(); if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = new MiniCluster(configBuilder.build()); + MiniCluster miniCluster = new MiniCluster(cfg); + try { - exec.start(); - return exec.executeJobBlocking(jobGraph); + miniCluster.start(); + configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); + + return miniCluster.executeJobBlocking(jobGraph); } finally { transformations.clear(); - exec.closeAsync(); + miniCluster.close(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 3c9f37811..01b3990de 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -194,14 +194,18 @@ private static void sqlTranslation(String localSqlPluginPath, SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); + + int scope = 0; for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, result, scope + ""); + scope++; } for (InsertSqlParser.SqlParseResult result : sqlTree.getExecSqlList()) { if (LOG.isInfoEnabled()) { LOG.info("exe-sql:\n" + result.getExecSql()); } + boolean isSide = false; for (String tableName : result.getTargetTableList()) { if (sqlTree.getTmpTableMap().containsKey(tableName)) { @@ -212,7 +216,7 @@ private static void sqlTranslation(String localSqlPluginPath, SqlNode sqlNode = flinkPlanner.parse(realSql); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); - sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp); + sideSqlExec.exec(tmp.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, tmp, scope + ""); } else { for (String sourceTable : result.getSourceTableList()) { if (sideTableMap.containsKey(sourceTable)) { @@ -222,7 +226,7 @@ private static void sqlTranslation(String localSqlPluginPath, } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null, null); } else { System.out.println("----------exec sql without dimension join-----------"); System.out.println("----------real sql exec is--------------------------"); @@ -233,6 +237,8 @@ private static void sqlTranslation(String localSqlPluginPath, } } } + + scope++; } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 8854ff4ec..997ed417e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.side; +import com.dtstack.flink.sql.util.TableUtils; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; @@ -66,6 +67,8 @@ public class JoinInfo implements Serializable { private JoinType joinType; + private String scope = ""; + /** * 左表需要查询的字段信息和output的时候对应的列名称 */ @@ -96,12 +99,14 @@ public String getNewTableName(){ //兼容左边表是as 的情况 String leftStr = leftTableName; leftStr = Strings.isNullOrEmpty(leftStr) ? leftTableAlias : leftStr; - return leftStr + "_" + rightTableName; + String newName = leftStr + "_" + rightTableName; + return TableUtils.buildTableNameWithScope(newName, scope); } public String getNewTableAlias(){ - return leftTableAlias + "_" + rightTableAlias; + String newName = leftTableAlias + "_" + rightTableAlias; + return TableUtils.buildTableNameWithScope(newName, scope); } public boolean isLeftIsSideTable() { @@ -233,6 +238,14 @@ public HashBasedTable getTableFieldRef(){ return mappingTable; } + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + @Override public String toString() { return "JoinInfo{" + diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 544e8ebb0..5a3111ffa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -89,7 +89,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, SqlNodeList parentGroupByList, Set> joinFieldSet, Map tableRef, - Map fieldRef) { + Map fieldRef, + String scope) { SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); @@ -105,13 +106,14 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, if (leftNode.getKind() == JOIN) { //处理连续join - dealNestJoin(joinNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); + dealNestJoin(joinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); leftNode = joinNode.getLeft(); } if (leftNode.getKind() == AS) { - AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, parentGroupByList, scope); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); } else if(leftNode.getKind() == IDENTIFIER){ @@ -122,7 +124,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, boolean leftIsSide = checkIsSideTable(leftTbName, sideTableSet); Preconditions.checkState(!leftIsSide, "side-table must be at the right of join operator"); - Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, parentGroupByList, scope); rightTableName = rightTableNameAndAlias.f0; rightTableAlias = rightTableNameAndAlias.f1; @@ -145,6 +148,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, tableInfo.setRightNode(rightNode); tableInfo.setJoinType(joinType); tableInfo.setCondition(joinNode.getCondition()); + tableInfo.setScope(scope); + TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); //extract 需要查询的字段信息 @@ -255,16 +260,21 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, SqlNodeList parentGroupByList, Set> joinFieldSet, Map tableRef, - Map fieldRef){ + Map fieldRef, + String scope){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); SqlNode rightNode = leftJoinNode.getRight(); - Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); - Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + + Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, + parentWhere, parentSelectList, parentGroupByList, scope); + Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, + queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); boolean parentRightIsSide = checkIsSideTable(parentRightJoinInfo.f0, sideTableSet); - JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); + JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); String rightTableName = rightTableNameAndAlias.f0; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); @@ -616,12 +626,13 @@ private void extractSelectField(SqlNode selectNode, private Tuple2 parseRightNode(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, - SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList) { + SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList, + String scope) { Tuple2 tabName = new Tuple2<>("", ""); if(sqlNode.getKind() == IDENTIFIER){ tabName.f0 = sqlNode.toString(); }else{ - AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList); + AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList, scope); tabName.f0 = aliasInfo.getName(); tabName.f1 = aliasInfo.getAlias(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index f37c3f78c..8187e2c02 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -61,7 +61,7 @@ public class SideSQLParser { private Map localTableCache = Maps.newHashMap(); - public Queue getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { + public Queue getExeQueue(String exeSql, Set sideTableSet, String scope) throws SqlParseException { System.out.println("----------exec original Sql----------"); System.out.println(exeSql); LOG.info("----------exec original Sql----------"); @@ -71,7 +71,7 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); - parseSql(sqlNode, sideTableSet, queueInfo, null, null, null); + parseSql(sqlNode, sideTableSet, queueInfo, null, null, null, scope); queueInfo.offer(sqlNode); return queueInfo; } @@ -91,7 +91,8 @@ public Object parseSql(SqlNode sqlNode, Queue queueInfo, SqlNode parentWhere, SqlNodeList parentSelectList, - SqlNodeList parentGroupByList){ + SqlNodeList parentGroupByList, + String scope){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ case WITH: { @@ -99,15 +100,15 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList sqlNodeList = sqlWith.withList; for (SqlNode withAsTable : sqlNodeList) { SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); queueInfo.add(sqlWithItem); } - parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); break; } case INSERT: SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); - return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); case SELECT: SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); SqlNode sqlWhere = ((SqlSelect)sqlNode).getWhere(); @@ -115,7 +116,7 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList groupByList = ((SqlSelect) sqlNode).getGroup(); if(sqlFrom.getKind() != IDENTIFIER){ - Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList); + Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList, scope); if(result instanceof JoinInfo){ return TableUtils.dealSelectResultWithJoinInfo((JoinInfo) result, (SqlSelect) sqlNode, queueInfo); }else if(result instanceof AliasInfo){ @@ -137,7 +138,7 @@ public Object parseSql(SqlNode sqlNode, Map tableRef = Maps.newHashMap(); Map fieldRef = Maps.newHashMap(); return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef); + parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); case AS: SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; @@ -146,7 +147,7 @@ public Object parseSql(SqlNode sqlNode, if(info.getKind() == IDENTIFIER){ infoStr = info.toString(); } else { - infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList).toString(); + infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope).toString(); } AliasInfo aliasInfo = new AliasInfo(); @@ -159,12 +160,12 @@ public Object parseSql(SqlNode sqlNode, SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); - parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); break; case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList); + parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); case LITERAL: return LITERAL.toString(); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index f90138b2a..9de8af04f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -101,7 +101,8 @@ public void exec(String sql, StreamTableEnvironment tableEnv, Map tableCache, StreamQueryConfig queryConfig, - CreateTmpTableParser.SqlParserResult createView) throws Exception { + CreateTmpTableParser.SqlParserResult createView, + String scope) throws Exception { if(localSqlPluginPath == null){ throw new RuntimeException("need to set localSqlPluginPath"); } @@ -121,7 +122,7 @@ public void exec(String sql, SideSQLParser sideSQLParser = new SideSQLParser(); sideSQLParser.setLocalTableCache(localTableCache); - Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); + Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet(), scope); Object pollObj = null; while((pollObj = exeQueue.poll()) != null){ @@ -452,9 +453,9 @@ private void joinFun(Object pollObj, replaceInfo.setTargetTableName(targetTableName); replaceInfo.setTargetTableAlias(targetTableAlias); - if (!tableEnv.isRegistered(joinInfo.getNewTableName())){ + if (!tableEnv.isRegistered(targetTableName)){ Table joinTable = tableEnv.fromDataStream(dsOut); - tableEnv.registerTable(joinInfo.getNewTableName(), joinTable); + tableEnv.registerTable(targetTableName, joinTable); localTableCache.put(joinInfo.getNewTableName(), joinTable); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 4a4973901..3075eb1df 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -198,11 +198,9 @@ public static SqlBasicCall buildAsNodeByJoinInfo(JoinInfo joinInfo, SqlNode sqlN SqlOperator operator = new SqlAsOperator(); SqlParserPos sqlParserPos = new SqlParserPos(0, 0); - String joinLeftTableName = joinInfo.getLeftTableName(); - String joinLeftTableAlias = joinInfo.getLeftTableAlias(); - joinLeftTableName = Strings.isNullOrEmpty(joinLeftTableName) ? joinLeftTableAlias : joinLeftTableName; - String newTableName = buildInternalTableName(joinLeftTableName, SPLIT, joinInfo.getRightTableName()); + String newTableName = joinInfo.getNewTableName(); String lefTbAlias = joinInfo.getLeftTableAlias(); + if(Strings.isNullOrEmpty(lefTbAlias)){ Set fromTableSet = Sets.newHashSet(); TableUtils.getFromTableInfo(joinInfo.getLeftNode(), fromTableSet); @@ -661,4 +659,12 @@ public static String buildTableField(String tableName, String fieldName){ } + public static String buildTableNameWithScope(String tableName, String scope){ + if(StringUtils.isEmpty(scope)){ + return tableName; + } + + return tableName + "_" + scope; + } + } From 7abe4dfd07757d026052915834329efbf6b53e6a Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 16:22:16 +0800 Subject: [PATCH 184/378] fix kudu and async quest --- .../sql/side/kudu/KuduAsyncSideInfo.java | 19 +++++++++++++++++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java index 5ace515f7..91b380594 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncSideInfo.java @@ -4,6 +4,9 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; +import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Lists; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -21,6 +24,22 @@ public KuduAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List sqlNodeList = Lists.newArrayList(); + ParseUtils.parseAnd(conditionNode, sqlNodeList); + + for (SqlNode sqlNode : sqlNodeList) { + dealOneEqualCon(sqlNode, sideTableName); + } + + sqlCondition = "select ${selectField} from ${tableName} "; + sqlCondition = sqlCondition.replace("${tableName}", kuduSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } @Override diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 7340357a9..7a6b45be6 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -155,13 +155,13 @@ private void connectWithRetry(Map inputParams, CRow input, Resul } }); try { - latch.wait(); + latch.await(); } catch (InterruptedException e) { LOG.error("", e); } if(!finishFlag.get()){ try { - Thread.sleep(100); + Thread.sleep(3000); } catch (Exception e){ LOG.error("", e); } From 3ca492b90a94a91749deaef71a91748fbd84bfea Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 15 May 2020 18:20:23 +0800 Subject: [PATCH 185/378] =?UTF-8?q?fix=20row=20=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=94=99=E4=B9=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 3 ++- .../com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 7e9937ad1..11240b973 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -160,7 +160,8 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ } @Override - public void asyncInvoke(CRow input, ResultFuture resultFuture) throws Exception { + public void asyncInvoke(CRow row, ResultFuture resultFuture) throws Exception { + CRow input = new CRow(Row.copy(row.row()), row.change()); preInvoke(input, resultFuture); Map inputParams = parseInputParam(input); if(MapUtils.isEmpty(inputParams)){ diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 71652042c..fbb0aa291 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -144,7 +144,7 @@ public void accept(Map values) { dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { - dealFillDataError(resultFuture, e, input); + dealFillDataError(input, resultFuture, e); } } else { dealMissKey(input, resultFuture); From 46598c486160b88ecee2584db8be3ef9a6bba459 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 18 May 2020 10:43:31 +0800 Subject: [PATCH 186/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E9=98=BB=E5=A1=9E=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clickhouse/ClickhouseAsyncReqRow.java | 6 ++++ .../flink/sql/side/db2/Db2AsyncReqRow.java | 6 ++++ .../sql/side/impala/ImpalaAsyncReqRow.java | 6 ++++ .../sql/side/mysql/MysqlAsyncReqRow.java | 6 ++++ .../sql/side/oracle/OracleAsyncReqRow.java | 6 ++++ .../sql/side/polardb/PolardbAsyncReqRow.java | 6 ++++ .../postgresql/PostgresqlAsyncReqRow.java | 6 ++++ .../sql/side/rdb/async/RdbAsyncReqRow.java | 28 +++++++++---------- .../side/sqlserver/SqlserverAsyncReqRow.java | 6 ++++ 9 files changed, 62 insertions(+), 14 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 9da7d8eaa..d279973dc 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.clickhouse; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { @@ -63,6 +67,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("clickhouseAsyncExec"))); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index 8e7275977..3a348df29 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.db2; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -33,6 +34,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Reason: @@ -75,6 +79,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("dbAsyncExec"))); } } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index b5e0ed5fb..3f2172f6d 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.impala; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -35,6 +36,9 @@ import java.io.IOException; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019/11/12 @@ -79,6 +83,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("impalaAsyncExec"))); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index d52f1918f..4ac7e8ba5 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.mysql; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Mysql dim table @@ -72,6 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("mysqlAsyncExec"))); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 938fd870c..53d6ff896 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.oracle; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class OracleAsyncReqRow extends RdbAsyncReqRow { @@ -64,5 +68,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("oracleAsyncExec"))); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 13d25b111..63b80ef0d 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -17,6 +17,7 @@ */ package com.dtstack.flink.sql.side.polardb; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -32,6 +33,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019/12/20 @@ -71,5 +75,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("polardbAsyncExec"))); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 42aa39a51..cedcb9eca 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.postgresql; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; @@ -34,6 +35,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019-08-11 @@ -72,6 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("postgresqlAsyncExec"))); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 7a6b45be6..e92ab282b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.side.rdb.async; import com.dtstack.flink.sql.enums.ECacheContentType; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.CacheMissVal; @@ -46,8 +47,7 @@ import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -84,6 +84,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private AtomicBoolean connectionStatus = new AtomicBoolean(true); + private transient ThreadPoolExecutor executor; + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); init(sideInfo); @@ -112,18 +114,7 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul Thread.sleep(100); } Map params = formatInputParam(inputParams); - rdbSqlClient.getConnection(conn -> { - if(conn.failed()){ - connectionStatus.set(false); - connectWithRetry(params, input, resultFuture, rdbSqlClient); - return; - } - connectionStatus.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), params, input, resultFuture); - }); - + executor.execute(() -> connectWithRetry(params, input, resultFuture, rdbSqlClient)); } private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { @@ -134,6 +125,7 @@ private void connectWithRetry(Map inputParams, CRow input, Resul rdbSqlClient.getConnection(conn -> { try { if(conn.failed()){ + connectionStatus.set(false); if(failCounter.getAndIncrement() % 1000 == 0){ LOG.error("getConnection error", conn.cause()); } @@ -210,12 +202,20 @@ public void close() throws Exception { rdbSqlClient.close(); } + if(executor != null){ + executor.shutdown(); + } + } public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } + public void setExecutor(ThreadPoolExecutor executor) { + this.executor = executor; + } + private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture){ String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index 1f4fbf13a..e8a22dfe8 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.side.sqlserver; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -34,6 +35,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Date: 2019/11/26 @@ -74,5 +78,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); + setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10), new DTThreadFactory("sqlServerAsyncExec"))); } } From 34cc5825fcd4b950cbcde4fa04376868365318df Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 19 May 2020 17:28:20 +0800 Subject: [PATCH 187/378] =?UTF-8?q?fix=20=E9=87=8D=E6=9E=84=E4=B9=8B?= =?UTF-8?q?=E5=90=8E=E7=9A=84=E7=BC=93=E5=AD=98=E6=A0=BC=E5=BC=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 0d8691d1c..094f5a6ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -203,7 +203,7 @@ private void invokeWithCache(Map inputParams, CRow input, Result return; }else if(ECacheContentType.SingleLine == val.getType()){ try { - Row row = fillData(input.row(), val); + Row row = fillData(input.row(), val.getContent()); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(input, resultFuture, e); From 312988833605330ac00c8461acd0639d86558239 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 14:44:44 +0800 Subject: [PATCH 188/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E5=92=8Credis=E6=96=87=E6=A1=A3=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSink.md | 3 ++- docs/pluginsInfo.md | 52 ++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index 224c26b2f..103cb4997 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -25,7 +25,6 @@ redis5.0 | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 | colName | 列名称，redis中存储为 表名:主键名:主键值:列名]| | colType | 列类型，当前只支持varchar| -| PRIMARY KEY(keyInfo) | 结果表主键定义;多个列之间用逗号隔开| ## 4.参数： @@ -44,6 +43,7 @@ redis5.0 |maxIdle|最大空闲连接数|否|8| |minIdle|最小空闲连接数|否||0| |masterName| 哨兵模式下的masterName|否|| +|primarykeys|主键字段，多个字段以逗号分割|是|| ## 5.样例： @@ -69,6 +69,7 @@ redis5.0 pv VARCHAR )WITH( type ='redis', + primarykeys='name', redisType ='1', url ='172.16.8.109:6379', tableName ='resultTable', diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 6baec3faf..60bfc7fc1 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,32 +1,32 @@ ### 1 插件列表 #### 1.1 源表插件 -* [kafka 源表插件](docs/plugin/kafkaSource.md) +* [kafka 源表插件](plugin/kafkaSource.md) #### 1.2 结果表插件 -* [elasticsearch 结果表插件](docs/plugin/elasticsearchSink.md) -* [hbase 结果表插件](docs/plugin/hbaseSink.md) -* [mysql 结果表插件](docs/plugin/mysqlSink.md) -* [oracle 结果表插件](docs/plugin/oracleSink.md) -* [mongo 结果表插件](docs/plugin/mongoSink.md) -* [redis 结果表插件](docs/plugin/redisSink.md) -* [cassandra 结果表插件](docs/plugin/cassandraSink.md) -* [kudu 结果表插件](docs/plugin/kuduSink.md) -* [postgresql 结果表插件](docs/plugin/postgresqlSink.md) -* [clickhouse 结果表插件](docs/plugin/clickhouseSink.md) -* [impala 结果表插件](docs/plugin/impalaSink.md) -* [db2 结果表插件](docs/plugin/db2Sink.md) -* [sqlserver 结果表插件](docs/plugin/sqlserverSink.md) +* [elasticsearch 结果表插件](plugin/elasticsearchSink.md) +* [hbase 结果表插件](plugin/hbaseSink.md) +* [mysql 结果表插件](plugin/mysqlSink.md) +* [oracle 结果表插件](plugin/oracleSink.md) +* [mongo 结果表插件](plugin/mongoSink.md) +* [redis 结果表插件](plugin/redisSink.md) +* [cassandra 结果表插件](plugin/cassandraSink.md) +* [kudu 结果表插件](plugin/kuduSink.md) +* [postgresql 结果表插件](plugin/postgresqlSink.md) +* [clickhouse 结果表插件](plugin/clickhouseSink.md) +* [impala 结果表插件](plugin/impalaSink.md) +* [db2 结果表插件](plugin/db2Sink.md) +* [sqlserver 结果表插件](plugin/sqlserverSink.md) #### 1.3 维表插件 -* [hbase 维表插件](docs/plugin/hbaseSide.md) -* [mysql 维表插件](docs/plugin/mysqlSide.md) -* [oracle 维表插件](docs/plugin/oracleSide.md) -* [mongo 维表插件](docs/plugin/mongoSide.md) -* [redis 维表插件](docs/plugin/redisSide.md) -* [cassandra 维表插件](docs/plugin/cassandraSide.md) -* [kudu 维表插件](docs/plugin/kuduSide.md) -* [postgresql 维表插件](docs/plugin/postgresqlSide.md) -* [clickhouse 维表插件](docs/plugin/clickhouseSide.md) -* [impala 维表插件](docs/plugin/impalaSide.md) -* [db2 维表插件](docs/plugin/db2Side.md) -* [sqlserver 维表插件](docs/plugin/sqlserverSide.md) +* [hbase 维表插件](plugin/hbaseSide.md) +* [mysql 维表插件](plugin/mysqlSide.md) +* [oracle 维表插件](plugin/oracleSide.md) +* [mongo 维表插件](plugin/mongoSide.md) +* [redis 维表插件](plugin/redisSide.md) +* [cassandra 维表插件](plugin/cassandraSide.md) +* [kudu 维表插件](plugin/kuduSide.md) +* [postgresql 维表插件](plugin/postgresqlSide.md) +* [clickhouse 维表插件](plugin/clickhouseSide.md) +* [impala 维表插件](plugin/impalaSide.md) +* [db2 维表插件](plugin/db2Side.md) +* [sqlserver 维表插件](plugin/sqlserverSide.md) From 85285e2a1cec9fc857c929b9b12b549e62101833 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 15:03:38 +0800 Subject: [PATCH 189/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E5=86=85=E9=83=A8=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/cassandraSink.md | 2 +- docs/plugin/clickhouseSide.md | 2 +- docs/plugin/consoleSink.md | 2 +- docs/plugin/db2Side.md | 2 +- docs/plugin/db2Sink.md | 2 +- docs/plugin/elasticsearch6Side.md | 2 +- docs/plugin/elasticsearch6Sink.md | 2 +- docs/plugin/elasticsearchSink.md | 2 +- docs/plugin/hbaseSide.md | 6 +++--- docs/plugin/hbaseSink.md | 2 +- docs/plugin/impalaSide.md | 2 +- docs/plugin/impalaSink.md | 2 +- docs/plugin/kafkaSink.md | 2 +- docs/plugin/kafkaSource.md | 4 ++-- docs/plugin/kuduSide.md | 4 ++-- docs/plugin/kuduSink.md | 2 +- docs/plugin/mongoSide.md | 2 +- docs/plugin/mongoSink.md | 2 +- docs/plugin/mysqlSide.md | 2 +- docs/plugin/mysqlSink.md | 2 +- docs/plugin/oracleSide.md | 2 +- docs/plugin/oracleSink.md | 2 +- docs/plugin/polardbSide.md | 2 +- docs/plugin/polardbSink.md | 2 +- docs/plugin/postgresqlSide.md | 2 +- docs/plugin/postgresqlSink.md | 2 +- docs/plugin/redisSide.md | 2 +- docs/plugin/sqlserverSide.md | 2 +- docs/plugin/sqlserverSink.md | 2 +- 29 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/plugin/cassandraSink.md b/docs/plugin/cassandraSink.md index 8702acb7a..a4ffaf818 100644 --- a/docs/plugin/cassandraSink.md +++ b/docs/plugin/cassandraSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数 diff --git a/docs/plugin/clickhouseSide.md b/docs/plugin/clickhouseSide.md index df050ca8b..c41226973 100644 --- a/docs/plugin/clickhouseSide.md +++ b/docs/plugin/clickhouseSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) clinkhose独有的参数信息 diff --git a/docs/plugin/consoleSink.md b/docs/plugin/consoleSink.md index d756f5a36..5a923236c 100644 --- a/docs/plugin/consoleSink.md +++ b/docs/plugin/consoleSink.md @@ -20,7 +20,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/db2Side.md b/docs/plugin/db2Side.md index af0c29c02..faf1b3060 100644 --- a/docs/plugin/db2Side.md +++ b/docs/plugin/db2Side.md @@ -28,7 +28,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) db2独有的参数配置： |参数名称|含义| diff --git a/docs/plugin/db2Sink.md b/docs/plugin/db2Sink.md index 9544e44f1..9994bc4a5 100644 --- a/docs/plugin/db2Sink.md +++ b/docs/plugin/db2Sink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| db2表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | primary key | updateMode为UPSERT时，需要指定的主键信息，不需要和数据库一致| diff --git a/docs/plugin/elasticsearch6Side.md b/docs/plugin/elasticsearch6Side.md index 62da6ca62..826d850b0 100644 --- a/docs/plugin/elasticsearch6Side.md +++ b/docs/plugin/elasticsearch6Side.md @@ -32,7 +32,7 @@ |----|---| | tableName | elasticsearch表名称| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型 [colType支持的类型](../colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| diff --git a/docs/plugin/elasticsearch6Sink.md b/docs/plugin/elasticsearch6Sink.md index 0f7a04280..4ecfd99f6 100644 --- a/docs/plugin/elasticsearch6Sink.md +++ b/docs/plugin/elasticsearch6Sink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| |tableName|在 sql 中使用的名称;即注册到flink-table-env上的名称| |colName|列名称| -|colType|列类型 [colType支持的类型](docs/colType.md)| +|colType|列类型 [colType支持的类型](../colType.md)| ## 4.参数： |参数名称|含义|是否必填|默认值| diff --git a/docs/plugin/elasticsearchSink.md b/docs/plugin/elasticsearchSink.md index 9a2a32b6f..99ac87ca5 100644 --- a/docs/plugin/elasticsearchSink.md +++ b/docs/plugin/elasticsearchSink.md @@ -22,7 +22,7 @@ CREATE TABLE tableName( |----|---| |tableName|在 sql 中使用的名称;即注册到flink-table-env上的名称| |colName|列名称| -|colType|列类型 [colType支持的类型](docs/colType.md)| +|colType|列类型 [colType支持的类型](../colType.md)| ## 4.参数： |参数名称|含义|是否必填|默认值| diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 25af638d6..ba1465aba 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -33,7 +33,7 @@ ## 4.参数 -参数详细说明请看[参数详细说明](./sideParams.md) +参数详细说明请看[参数详细说明](sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| @@ -138,7 +138,7 @@ CREATE TABLE MyResult( CREATE TABLE sideTable ( wtz:message varchar as message, wtz:info varchar as info , - PRIMARY KEY (rowkey), + PRIMARY KEY (md5(key1) + key2), PERIOD FOR SYSTEM_TIME ) WITH ( type = 'hbase', @@ -166,6 +166,6 @@ into MyTable a left join sideTable b - on a.id=b.rowkey; + on a.id=b.key1 and a.xx = b.key2; ``` diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index 00c340b3d..ef1be339b 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -24,7 +24,7 @@ hbase2.0 |----|---| | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 | colFamily:colName | hbase中的列族名称和列名称 -| colType | 列类型 [colType支持的类型](docs/colType.md) +| colType | 列类型 [colType支持的类型](../colType.md) ## 4.参数： diff --git a/docs/plugin/impalaSide.md b/docs/plugin/impalaSide.md index dce959388..64c22b498 100644 --- a/docs/plugin/impalaSide.md +++ b/docs/plugin/impalaSide.md @@ -26,7 +26,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) impala独有的参数配置 diff --git a/docs/plugin/impalaSink.md b/docs/plugin/impalaSink.md index 398ec6ef2..4847cf2e2 100644 --- a/docs/plugin/impalaSink.md +++ b/docs/plugin/impalaSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/plugin/impalaColType.md)| +| colType | 列类型 [colType支持的类型](impalaColType.md)| ## 4.参数： diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 8b50c9e3f..1d8d8ce5f 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -27,7 +27,7 @@ CREATE TABLE tableName( |----|---| | tableName| 结果表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index bfc182b79..9e628f287 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -29,7 +29,7 @@ CREATE TABLE tableName( |----|---| | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | function(colNameX) as aliasName | 支持在定义列信息的时候根据已有列类型生成新的列(函数可以使用系统函数和已经注册的UDF)| | WATERMARK FOR colName AS withOffset( colName , delayTime ) | 标识输入流生的watermake生成规则,根据指定的colName(当前支持列的类型为Long | Timestamp) 和delayTime生成waterMark 同时会在注册表的使用附带上rowtime字段(如果未指定则默认添加proctime字段);注意：添加该标识的使用必须设置系统参数 time.characteristic:EventTime; delayTime: 数据最大延迟时间(ms)| @@ -48,7 +48,7 @@ CREATE TABLE tableName( |sourcedatatype | 数据类型,avro,csv,json,dt_nest。dt_nest为默认JSON解析器，能够解析嵌套JSON数据类型，其他仅支持非嵌套格式|否|dt_nest| |schemaInfo | avro类型使用的schema信息|否|| |fieldDelimiter |csv类型使用的数据分隔符|否| | | -|timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' +|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' **kafka相关参数可以自定义，使用kafka.开头即可。** ``` kafka.consumer.id diff --git a/docs/plugin/kuduSide.md b/docs/plugin/kuduSide.md index a45300deb..a2636d4aa 100644 --- a/docs/plugin/kuduSide.md +++ b/docs/plugin/kuduSide.md @@ -36,13 +36,13 @@ |----|---| | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| + | colType | 列类型 [colType支持的类型](../colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| ## 4.参数 -参数详细说明请看[参数详细说明](./sideParams.md) +参数详细说明请看[参数详细说明](sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|-----| diff --git a/docs/plugin/kuduSink.md b/docs/plugin/kuduSink.md index b39ba1d67..cc8f8535a 100644 --- a/docs/plugin/kuduSink.md +++ b/docs/plugin/kuduSink.md @@ -27,7 +27,7 @@ kudu 1.9.0+cdh6.2.0 |----|---| | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 | colName | 列名称，redis中存储为 表名:主键名:主键值:列名]| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/mongoSide.md b/docs/plugin/mongoSide.md index 92362a752..9ff52b85c 100644 --- a/docs/plugin/mongoSide.md +++ b/docs/plugin/mongoSide.md @@ -26,7 +26,7 @@ ## 3.表结构定义 - [通用维表参数信息](docs/plugin/sideParams.md) + [通用维表参数信息](sideParams.md) mongo相关参数配置： diff --git a/docs/plugin/mongoSink.md b/docs/plugin/mongoSink.md index bf4bbb233..9eea6fd89 100644 --- a/docs/plugin/mongoSink.md +++ b/docs/plugin/mongoSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/mysqlSide.md b/docs/plugin/mysqlSide.md index 834183ed2..a4247ef0a 100644 --- a/docs/plugin/mysqlSide.md +++ b/docs/plugin/mysqlSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) mysql独有的参数配置： diff --git a/docs/plugin/mysqlSink.md b/docs/plugin/mysqlSink.md index ffc62792c..d54c36b28 100644 --- a/docs/plugin/mysqlSink.md +++ b/docs/plugin/mysqlSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 结果表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/oracleSide.md b/docs/plugin/oracleSide.md index b681536f8..feb99a1ee 100644 --- a/docs/plugin/oracleSide.md +++ b/docs/plugin/oracleSide.md @@ -31,7 +31,7 @@ 10g 11g ## 3. 表结构定义 - [维表参数](docs/plugin/sideParams.md) + [维表参数](sideParams.md) oracle独有的参数配置： diff --git a/docs/plugin/oracleSink.md b/docs/plugin/oracleSink.md index 93771f3d6..40c5836e9 100644 --- a/docs/plugin/oracleSink.md +++ b/docs/plugin/oracleSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| oracle表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/polardbSide.md b/docs/plugin/polardbSide.md index 625613f98..eaef0acb9 100644 --- a/docs/plugin/polardbSide.md +++ b/docs/plugin/polardbSide.md @@ -28,7 +28,7 @@ |----|---| | tableName | polardb表名称| | colName | 列名称| - | colType | 列类型 [colType支持的类型](docs/colType.md)| + | colType | 列类型 [colType支持的类型](../colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| diff --git a/docs/plugin/polardbSink.md b/docs/plugin/polardbSink.md index d252dd712..4a17b72c6 100644 --- a/docs/plugin/polardbSink.md +++ b/docs/plugin/polardbSink.md @@ -24,7 +24,7 @@ CREATE TABLE tableName( |----|---| | tableName| polardb表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： diff --git a/docs/plugin/postgresqlSide.md b/docs/plugin/postgresqlSide.md index b50a2de52..43fdffeee 100644 --- a/docs/plugin/postgresqlSide.md +++ b/docs/plugin/postgresqlSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) postgresql独有的参数配置 diff --git a/docs/plugin/postgresqlSink.md b/docs/plugin/postgresqlSink.md index eb055cc04..bc9f98fc0 100644 --- a/docs/plugin/postgresqlSink.md +++ b/docs/plugin/postgresqlSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | primary key | updateMode为UPSERT时，需要指定的主键信息| diff --git a/docs/plugin/redisSide.md b/docs/plugin/redisSide.md index c63c542cd..5a037c763 100644 --- a/docs/plugin/redisSide.md +++ b/docs/plugin/redisSide.md @@ -36,7 +36,7 @@ ## 4.参数 -参数详细说明请看[参数详细说明](./sideParams.md) +参数详细说明请看[参数详细说明](sideParams.md) |参数名称|含义|是否必填|默认值| |----|---|---|----| diff --git a/docs/plugin/sqlserverSide.md b/docs/plugin/sqlserverSide.md index f95d8bba5..90d036fd2 100644 --- a/docs/plugin/sqlserverSide.md +++ b/docs/plugin/sqlserverSide.md @@ -29,7 +29,7 @@ ## 3.表结构定义 - [维表参数信息](docs/plugin/sideParams.md) + [维表参数信息](sideParams.md) sqlserver独有的参数配置 diff --git a/docs/plugin/sqlserverSink.md b/docs/plugin/sqlserverSink.md index 0ab9431e9..4b82898fd 100644 --- a/docs/plugin/sqlserverSink.md +++ b/docs/plugin/sqlserverSink.md @@ -25,7 +25,7 @@ CREATE TABLE tableName( |----|---| | tableName| sqlserver表名称| | colName | 列名称| -| colType | 列类型 [colType支持的类型](docs/colType.md)| +| colType | 列类型 [colType支持的类型](../colType.md)| | primary key | updateMode为UPSERT时，需要指定的主键信息| ## 4.参数： From c83dc91a9c68ae087203371dac62ab1393074b1c Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 16:19:44 +0800 Subject: [PATCH 190/378] =?UTF-8?q?hbase=20=E8=AF=A6=E7=BB=86=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/hbaseSide.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index ba1465aba..e590b02e7 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -98,12 +98,12 @@ CREATE TABLE sideTable ( ``` hbase(main):002:0> scan 'testFlinkStreamSql' ROW COLUMN+CELL - 0 column=wtz:info, timestamp=1587089266719, value=hadoop - 0 column=wtz:message, timestamp=1587089245780, value=hbase - 1 column=wtz:info, timestamp=1587088818432, value=flink - 1 column=wtz:message, timestamp=1587088796633, value=dtstack - 2 column=wtz:info, timestamp=1587088858564, value=sql - 2 column=wtz:message, timestamp=1587088840507, value=stream + cfcd208495d565ef66e7dff9f98764dazhangsantest column=wtz:info, timestamp=1587089266719, value=hadoop + cfcd208495d565ef66e7dff9f98764dazhangsantest column=wtz:message, timestamp=1587089245780, value=hbase + c4ca4238a0b923820dcc509a6f75849blisitest column=wtz:info, timestamp=1587088818432, value=flink + c4ca4238a0b923820dcc509a6f75849blisitest column=wtz:message, timestamp=1587088796633, value=dtstack + c81e728d9d4c2f636f067f89cc14862cwangwutest column=wtz:info, timestamp=1587088858564, value=sql + c81e728d9d4c2f636f067f89cc14862cwangwutest column=wtz:message, timestamp=1587088840507, value=stream ``` 在hbase中，rowKey是一个二进制码流，可以为任意字符串，flinkStreamSql读取rowKey并通过rowKey唯一确定数据，对rowKey没有任何限制,对rowKey可选择的构造包括 md5(alias + alias), '常量',也可以它们的自由组合。 在本次案例中，rowKey为了简单，设置成了"0,1,2"这样的数值型字符，若有必要，也可以设计得更为复杂。 @@ -138,7 +138,7 @@ CREATE TABLE MyResult( CREATE TABLE sideTable ( wtz:message varchar as message, wtz:info varchar as info , - PRIMARY KEY (md5(key1) + key2), + PRIMARY KEY (md5(rowkey1) + rowkey2 + 'test'), PERIOD FOR SYSTEM_TIME ) WITH ( type = 'hbase', @@ -166,6 +166,6 @@ into MyTable a left join sideTable b - on a.id=b.key1 and a.xx = b.key2; + on a.id=b.rowkey1 and a.name = b.rowkey2; ``` From 33047f6a8f2cee0ed59b9a65e8bf5b7a1f15f8a1 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 20:00:40 +0800 Subject: [PATCH 191/378] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E4=B9=B1=E5=90=97=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 16 ++++++++++++++-- .../flink/sql/sink/impala/ImpalaSink.java | 4 ++-- .../sql/sink/impala/table/ImpalaSinkParser.java | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 3eaffc0ad..89fb769c3 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; import java.util.Arrays; import java.util.List; @@ -38,6 +39,12 @@ public class ImpalaDialect implements JDBCDialect { private static final String IMPALA_PARTITION_KEYWORD = "partition"; + private TypeInformation[] fieldTypes; + + public ImpalaDialect(TypeInformation[] fieldTypes){ + this.fieldTypes = fieldTypes; + } + @Override public boolean canHandle(String url) { return url.startsWith("jdbc:impala:"); @@ -70,8 +77,13 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(this::quoteIdentifier) .collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fieldNames) - .map(f -> "?") + String placeholders = Arrays.stream(fieldTypes) + .map(f -> { + if(String.class.getName().equals(f.getTypeClass().getName())){ + return "cast( ? as string)"; + } + return "?"; + }) .collect(Collectors.joining(", ")); String partitionFieldStr = partitionFieldsList.stream() diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index cd1e1e945..910f6f4c5 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -41,14 +41,14 @@ public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener Date: Mon, 25 May 2020 14:51:51 +0800 Subject: [PATCH 192/378] add hbase kerbos --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 85 ++++++++++++-- .../sql/side/hbase/HbaseAsyncReqRow.java | 40 ++++++- .../side/hbase/table/HbaseSideTableInfo.java | 48 ++++++++ .../side/hbase/utils/HbaseConfigUtils.java | 108 ++++++++++++++++++ 4 files changed, 272 insertions(+), 9 deletions(-) create mode 100644 hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 63d26d5dd..401ef00c0 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,8 +25,10 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; +import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; @@ -36,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -44,10 +47,13 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; @@ -166,13 +172,44 @@ public void flatMap(CRow input, Collector out) throws Exception { private void loadData(Map> tmpCache) throws SQLException { AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; - Configuration conf = new Configuration(); - conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); - Connection conn = null; - Table table = null; - ResultScanner resultScanner = null; + boolean openKerberos = hbaseSideTableInfo.isKerberosAuthEnable(); + int loadDataCount = 0; try { - conn = ConnectionFactory.createConnection(conf); + conf = HBaseConfiguration.create(); + if (openKerberos) { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); + + fillSyncKerberosConfig(conf,hbaseSideTableInfo); + LOG.info("hbase.security.authentication:{}", conf.get("hbase.security.authentication")); + LOG.info("hbase.security.authorization:{}", conf.get("hbase.security.authorization")); + LOG.info("hbase.master.keytab.file:{}", conf.get("hbase.master.keytab.file")); + LOG.info("hbase.master.kerberos.principal:{}", conf.get("hbase.master.kerberos.principal")); + LOG.info("hbase.regionserver.keytab.file:{}", conf.get("hbase.regionserver.keytab.file")); + LOG.info("hbase.regionserver.kerberos.principal:{}", conf.get("hbase.regionserver.kerberos.principal")); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, hbaseSideTableInfo.getRegionserverPrincipal(), + hbaseSideTableInfo.getRegionserverKeytabFile()); + + Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + + } else { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); + conn = ConnectionFactory.createConnection(conf); + } + table = conn.getTable(TableName.valueOf(tableName)); resultScanner = table.getScanner(new Scan()); for (Result r : resultScanner) { @@ -187,13 +224,15 @@ private void loadData(Map> tmpCache) throws SQLExcep kv.put(aliasNameInversion.get(key.toString()), value); } + loadDataCount++; tmpCache.put(new String(r.getRow()), kv); } } catch (IOException e) { - LOG.error("", e); + throw new RuntimeException(e); } finally { + LOG.info("load Data count: {}", loadDataCount); try { - if (null != conn && !conn.isClosed()) { + if (null != conn) { conn.close(); } @@ -209,4 +248,34 @@ private void loadData(Map> tmpCache) throws SQLExcep } } } + + private void fillSyncKerberosConfig(Configuration config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { + String regionserverKeytabFile = hbaseSideTableInfo.getRegionserverKeytabFile(); + if (StringUtils.isEmpty(regionserverKeytabFile)) { + throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); + } + String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; + LOG.info("regionserverKeytabFilePath:{}", regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); + + String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); + } + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } } \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 56f50e27c..d43d1408a 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -31,6 +31,7 @@ import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.commons.lang3.StringUtils; @@ -40,10 +41,13 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import org.hbase.async.Config; import org.hbase.async.HBaseClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; import java.sql.Timestamp; import java.util.Collections; import java.util.List; @@ -93,11 +97,19 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; + ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); - hBaseClient = new HBaseClient(hbaseSideTableInfo.getHost(), hbaseSideTableInfo.getParent(), executorService); + Config config = new Config(); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC, hbaseSideTableInfo.getParent()); + + if (hbaseSideTableInfo.isKerberosAuthEnable()) { + fillAsyncKerberosConfig(config, hbaseSideTableInfo); + } + hBaseClient = new HBaseClient(config, executorService); try { Deferred deferred = hBaseClient.ensureTableExists(tableName) @@ -166,6 +178,32 @@ public void close() throws Exception { hBaseClient.shutdown(); } + private void fillAsyncKerberosConfig(Config config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { + AuthUtil.JAASConfig jaasConfig = HbaseConfigUtils.buildJaasConfig(hbaseSideTableInfo); + LOG.info("jaasConfig file:\n {}", jaasConfig.toString()); + String jaasFilePath = AuthUtil.creatJaasFile("JAAS", ".conf", jaasConfig); + config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTH_ENABLE, "true"); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SASL_CLIENTCONFIG, "Client"); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, regionserverPrincipal); + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); + } + + if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } + class CheckResult{ diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index 3cedb2c68..d8b0a62bd 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -148,6 +148,54 @@ public void setPreRowKey(boolean preRowKey) { this.preRowKey = preRowKey; } + public boolean isKerberosAuthEnable() { + return kerberosAuthEnable; + } + + public void setKerberosAuthEnable(boolean kerberosAuthEnable) { + this.kerberosAuthEnable = kerberosAuthEnable; + } + + public String getRegionserverKeytabFile() { + return regionserverKeytabFile; + } + + public void setRegionserverKeytabFile(String regionserverKeytabFile) { + this.regionserverKeytabFile = regionserverKeytabFile; + } + + public String getRegionserverPrincipal() { + return regionserverPrincipal; + } + + public void setRegionserverPrincipal(String regionserverPrincipal) { + this.regionserverPrincipal = regionserverPrincipal; + } + + public String getJaasPrincipal() { + return jaasPrincipal; + } + + public void setJaasPrincipal(String jaasPrincipal) { + this.jaasPrincipal = jaasPrincipal; + } + + public String getSecurityKrb5Conf() { + return securityKrb5Conf; + } + + public void setSecurityKrb5Conf(String securityKrb5Conf) { + this.securityKrb5Conf = securityKrb5Conf; + } + + public String getZookeeperSaslClient() { + return zookeeperSaslClient; + } + + public void setZookeeperSaslClient(String zookeeperSaslClient) { + this.zookeeperSaslClient = zookeeperSaslClient; + } + @Override public void finish(){ super.finish(); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java new file mode 100644 index 000000000..09f2ea53d --- /dev/null +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.hbase.utils; + +import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.util.AuthUtil; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * + * The utility class of HBase connection + * + * Date: 2019/12/24 + * Company: www.dtstack.com + * @author maqi + */ +public class HbaseConfigUtils { + + private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); + // sync side kerberos + public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + // async side kerberos + public final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; + public final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; + public final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC = "zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC = "hbase.zookeeper.znode.parent"; + + + public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; + + public static final String KEY_JAVA_SECURITY_AUTH_LOGIN_CONF = "java.security.auth.login.config"; + + + public static AuthUtil.JAASConfig buildJaasConfig(HbaseSideTableInfo hbaseSideTableInfo) { + String keytabPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getRegionserverKeytabFile(); + Map loginModuleOptions = new HashMap<>(); + loginModuleOptions.put("useKeyTab", "true"); + loginModuleOptions.put("useTicketCache", "false"); + loginModuleOptions.put("keyTab", "\"" + keytabPath + "\""); + loginModuleOptions.put("principal", "\"" + hbaseSideTableInfo.getJaasPrincipal() + "\""); + return AuthUtil.JAASConfig.builder().setEntryName("Client") + .setLoginModule("com.sun.security.auth.module.Krb5LoginModule") + .setLoginModuleFlag("required").setLoginModuleOptions(loginModuleOptions).build(); + } + + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + if (conf == null) { + throw new IllegalArgumentException("kerberos conf can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(principal)) { + throw new IllegalArgumentException("principal can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(keytab)) { + throw new IllegalArgumentException("keytab can not be null"); + } + + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + } + +} From 1e3237f503754c38d4b89b6ee28d2473d8bf904b Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 14:54:27 +0800 Subject: [PATCH 193/378] fix missing class --- .../com/dtstack/flink/sql/util/AuthUtil.java | 112 ++++++++++++++++++ .../flink/sql/side/hbase/HbaseAllReqRow.java | 4 + .../sql/side/hbase/HbaseAsyncReqRow.java | 1 + 3 files changed, 117 insertions(+) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java new file mode 100644 index 000000000..646a11b66 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/AuthUtil.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Utility methods for helping with security tasks. + * Date: 2019/12/28 + * Company: www.dtstack.com + * @author maqi + */ +public class AuthUtil { + + public static String creatJaasFile(String prefix, String suffix, JAASConfig jaasConfig) throws IOException { + File krbConf = new File(System.getProperty("user.dir")); + File temp = File.createTempFile(prefix, suffix, krbConf); + temp.deleteOnExit(); + FileUtils.writeStringToFile(temp, jaasConfig.toString()); + return temp.getAbsolutePath(); + } + + + public static class JAASConfig { + private String entryName; + private String loginModule; + private String loginModuleFlag; + private Map loginModuleOptions; + + public JAASConfig(String entryName, String loginModule, String loginModuleFlag, Map loginModuleOptions) { + this.entryName = entryName; + this.loginModule = loginModule; + this.loginModuleFlag = loginModuleFlag; + this.loginModuleOptions = loginModuleOptions; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(entryName).append(" {\n\t") + .append(loginModule).append(" ").append(loginModuleFlag).append("\n\t"); + String[] keys = loginModuleOptions.keySet().toArray(new String[loginModuleOptions.size()]); + for (int i = 0; i < keys.length; i++) { + stringBuilder.append(keys[i]).append("=").append(loginModuleOptions.get(keys[i])); + if (i != keys.length - 1) { + stringBuilder.append("\n\t"); + } else { + stringBuilder.append(";\n"); + } + + } + stringBuilder.append("\n").append("};"); + return stringBuilder.toString(); + } + + public static class Builder { + private String entryName; + private String loginModule; + private String loginModuleFlag; + private Map loginModuleOptions; + + public Builder setEntryName(String entryName) { + this.entryName = entryName; + return this; + } + + public Builder setLoginModule(String loginModule) { + this.loginModule = loginModule; + return this; + } + + public Builder setLoginModuleFlag(String loginModuleFlag) { + this.loginModuleFlag = loginModuleFlag; + return this; + } + + public Builder setLoginModuleOptions(Map loginModuleOptions) { + this.loginModuleOptions = loginModuleOptions; + return this; + } + + public JAASConfig build() { + return new JAASConfig( + entryName, loginModule, loginModuleFlag, loginModuleOptions); + } + } + } +} diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 401ef00c0..c50e1a806 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -71,6 +71,10 @@ public class HbaseAllReqRow extends BaseAllReqRow { private Map aliasNameInversion; private AtomicReference>> cacheRef = new AtomicReference<>(); + private Connection conn = null; + private Table table = null; + private ResultScanner resultScanner = null; + private Configuration conf = null; public HbaseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new HbaseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index d43d1408a..f3a1d6691 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -32,6 +32,7 @@ import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; +import com.dtstack.flink.sql.util.AuthUtil; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.commons.lang3.StringUtils; From 6bb89495cc5679bd4eea8a8ce5166472a6bf6e64 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 14:56:59 +0800 Subject: [PATCH 194/378] fill missing field --- .../sql/side/hbase/table/HbaseSideTableInfo.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index d8b0a62bd..2a5411682 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -52,6 +52,18 @@ public class HbaseSideTableInfo extends AbstractSideTableInfo { private String tableName; + private boolean kerberosAuthEnable; + + private String regionserverKeytabFile; + + private String regionserverPrincipal; + + private String jaasPrincipal; + + private String securityKrb5Conf; + + private String zookeeperSaslClient; + private String[] columnRealNames; private List columnRealNameList = Lists.newArrayList(); From 907aacb8961d6521c1b91f0fe1b1651318fc2a1b Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 15:15:36 +0800 Subject: [PATCH 195/378] hbase kerberos sink --- .../sql/sink/hbase/HbaseOutputFormat.java | 33 +++++++++++++++- .../flink/sql/sink/hbase/HbaseSink.java | 39 +++++++++++++++---- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a3189b0a6..faea0845c 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -62,6 +62,12 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String[] columnTypes; private Map columnNameFamily; + private boolean kerberosAuthEnable; + private String regionserverKeytabFile; + private String regionserverPrincipal; + private String securityKrb5Conf; + private String zookeeperSaslClient; + private String[] families; private String[] qualifiers; @@ -252,6 +258,31 @@ public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNa format.columnNameFamily = columnNameFamily; return this; } + public HbaseOutputFormatBuilder setKerberosAuthEnable(boolean kerberosAuthEnable) { + format.kerberosAuthEnable = kerberosAuthEnable; + return this; + } + + public HbaseOutputFormatBuilder setRegionserverKeytabFile(String regionserverKeytabFile) { + format.regionserverKeytabFile = regionserverKeytabFile; + return this; + } + + public HbaseOutputFormatBuilder setRegionserverPrincipal(String regionserverPrincipal) { + format.regionserverPrincipal = regionserverPrincipal; + return this; + } + + public HbaseOutputFormatBuilder setSecurityKrb5Conf(String securityKrb5Conf) { + format.securityKrb5Conf = securityKrb5Conf; + return this; + } + + public HbaseOutputFormatBuilder setZookeeperSaslClient(String zookeeperSaslClient) { + format.zookeeperSaslClient = zookeeperSaslClient; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); @@ -267,7 +298,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = StringUtils.split(col, ":"); + String[] part = col.split(":"); families[i] = part[0]; qualifiers[i] = part[1]; } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 13bd98b70..8364b6c10 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -52,6 +52,15 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String tableName; protected String updateMode; protected String rowkey; + protected String registerTabName; + + protected boolean kerberosAuthEnable; + protected String regionserverKeytabFile; + protected String regionserverPrincipal; + protected String securityKrb5Conf; + protected String zookeeperSaslClient; + private int parallelism = -1; + public HbaseSink() { // TO DO NOTHING @@ -66,20 +75,34 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.tableName = hbaseTableInfo.getTableName(); this.rowkey = hbaseTableInfo.getRowkey(); this.columnNameFamily = hbaseTableInfo.getColumnNameFamily(); - this.updateMode = hbaseTableInfo.getUpdateMode(); + this.registerTabName = hbaseTableInfo.getName(); + + this.kerberosAuthEnable = hbaseTableInfo.isKerberosAuthEnable(); + this.regionserverKeytabFile = hbaseTableInfo.getRegionserverKeytabFile(); + this.regionserverPrincipal = hbaseTableInfo.getRegionserverPrincipal(); + this.securityKrb5Conf = hbaseTableInfo.getSecurityKrb5Conf(); + this.zookeeperSaslClient = hbaseTableInfo.getZookeeperSaslClient(); + + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); + if (tmpSinkParallelism != null) { + this.parallelism = tmpSinkParallelism; + } return this; } @Override public void emitDataStream(DataStream> dataStream) { HbaseOutputFormat.HbaseOutputFormatBuilder builder = HbaseOutputFormat.buildHbaseOutputFormat(); - builder.setHost(this.zookeeperQuorum) - .setZkParent(this.parent) - .setTable(this.tableName) - .setRowkey(rowkey) - .setUpdateMode(updateMode) - .setColumnNames(fieldNames) - .setColumnNameFamily(columnNameFamily); + builder.setHost(this.zookeeperQuorum).setZkParent(this.parent).setTable(this.tableName); + + builder.setRowkey(rowkey); + builder.setColumnNames(fieldNames); + builder.setColumnNameFamily(columnNameFamily); + builder.setKerberosAuthEnable(kerberosAuthEnable); + builder.setRegionserverKeytabFile(regionserverKeytabFile); + builder.setRegionserverPrincipal(regionserverPrincipal); + builder.setSecurityKrb5Conf(securityKrb5Conf); + builder.setZookeeperSaslClient(zookeeperSaslClient); HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); From 3a9cb360d3d6c7e54e75170c1ae638c3e154d428 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 15:18:21 +0800 Subject: [PATCH 196/378] fix missing field --- .../sql/sink/hbase/table/HbaseTableInfo.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index 36f8e89e5..49df2f757 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -23,6 +23,8 @@ import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + import java.util.Map; /** @@ -54,6 +56,18 @@ public class HbaseTableInfo extends AbstractTargetTableInfo { private String updateMode; + private boolean kerberosAuthEnable; + + private String regionserverKeytabFile; + + private String regionserverPrincipal; + + private String securityKrb5Conf; + + private String zookeeperSaslClient; + + private Map hbaseConfig = Maps.newHashMap(); + public HbaseTableInfo(){ setType(CURR_TYPE); } @@ -149,4 +163,53 @@ public String getType() { return super.getType().toLowerCase(); } + + public Map getHbaseConfig() { + return hbaseConfig; + } + + public void setHbaseConfig(Map hbaseConfig) { + this.hbaseConfig = hbaseConfig; + } + + public boolean isKerberosAuthEnable() { + return kerberosAuthEnable; + } + + public void setKerberosAuthEnable(boolean kerberosAuthEnable) { + this.kerberosAuthEnable = kerberosAuthEnable; + } + + public String getRegionserverKeytabFile() { + return regionserverKeytabFile; + } + + public void setRegionserverKeytabFile(String regionserverKeytabFile) { + this.regionserverKeytabFile = regionserverKeytabFile; + } + + public String getRegionserverPrincipal() { + return regionserverPrincipal; + } + + public void setRegionserverPrincipal(String regionserverPrincipal) { + this.regionserverPrincipal = regionserverPrincipal; + } + + public String getSecurityKrb5Conf() { + return securityKrb5Conf; + } + + public void setSecurityKrb5Conf(String securityKrb5Conf) { + this.securityKrb5Conf = securityKrb5Conf; + } + + public String getZookeeperSaslClient() { + return zookeeperSaslClient; + } + + public void setZookeeperSaslClient(String zookeeperSaslClient) { + this.zookeeperSaslClient = zookeeperSaslClient; + } + } From 2f0fdfeec8ddec660618e3574c77226d2b9fc7a4 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 25 May 2020 17:46:25 +0800 Subject: [PATCH 197/378] =?UTF-8?q?hbase=20kerberos=20=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/hbase/table/HbaseSideParser.java | 14 ++++++++++++++ .../sql/sink/hbase/table/HbaseSinkParser.java | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 80753c40b..832961834 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -53,6 +53,13 @@ public class HbaseSideParser extends AbstractSideTableParser { public static final String CACHE = "cache"; + public static final String KERBEROS_AUTH_ENABLE_KEY = "kerberosAuthEnable"; + public static final String REGIONSERVER_KEYTAB_FILE_KEY = "regionserverKeytabFile"; + public static final String REGIONSERVER_PRINCIPAL_KEY = "regionserverPrincipal"; + public static final String JAAS_PRINCIPAL_KEY = "jaasPrincipal"; + public static final String SECURITY_KRB5_CONF_KEY = "securityKrb5Conf"; + public static final String ZOOKEEPER_SASL_CLINT_KEY = "zookeeperSaslClient"; + public HbaseSideParser() { addParserHandler(FIELD_KEY, FIELD_PATTERN, this::dealField); } @@ -69,6 +76,13 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 25 May 2020 17:56:06 +0800 Subject: [PATCH 198/378] =?UTF-8?q?=E4=BF=9D=E6=8C=81=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index faea0845c..3def5cb1d 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -298,7 +298,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = col.split(":"); + String[] part = StringUtils.split(col, ":");; families[i] = part[0]; qualifiers[i] = part[1]; } From a9f9299d9b1d9624e921c0f8cce7f296d2b9ea60 Mon Sep 17 00:00:00 2001 From: whiletrue Date: Tue, 26 May 2020 20:04:34 +0800 Subject: [PATCH 199/378] Create SECURITY.md --- SECURITY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..034e84803 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 5.1.x | :white_check_mark: | +| 5.0.x | :x: | +| 4.0.x | :white_check_mark: | +| < 4.0 | :x: | + +## Reporting a Vulnerability + +Use this section to tell people how to report a vulnerability. + +Tell them where to go, how often they can expect to get an update on a +reported vulnerability, what to expect if the vulnerability is accepted or +declined, etc. From 9d80379c90be792d35bdd9ecef20713e72dcb49f Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 10:56:38 +0800 Subject: [PATCH 200/378] =?UTF-8?q?fix=20=E8=A1=A5=E5=85=85=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseConfigUtils.java | 79 +++++++++++++++++++ .../sql/sink/hbase/HbaseOutputFormat.java | 56 +++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java new file mode 100644 index 000000000..2be03484b --- /dev/null +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.hbase; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * + * The utility class of HBase connection + * + * Date: 2019/12/24 + * Company: www.dtstack.com + * @author maqi + */ +public class HbaseConfigUtils { + + private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); + // sync side kerberos + public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + + public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + + + public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + if (conf == null) { + throw new IllegalArgumentException("kerberos conf can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(principal)) { + throw new IllegalArgumentException("principal can not be null"); + } + + if (org.apache.commons.lang.StringUtils.isEmpty(keytab)) { + throw new IllegalArgumentException("keytab can not be null"); + } + + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + } +} diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 3def5cb1d..36b251d14 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -37,10 +37,13 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.security.PrivilegedAction; import java.util.List; import java.util.Map; import java.util.Set; @@ -91,6 +94,29 @@ public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); conn = ConnectionFactory.createConnection(conf); table = conn.getTable(TableName.valueOf(tableName)); + if (kerberosAuthEnable) { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); + org.apache.hadoop.conf.Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + } else { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + conn = ConnectionFactory.createConnection(conf); + } LOG.warn("---open end(get table from hbase) ---"); initMetric(); } @@ -311,5 +337,35 @@ public HbaseOutputFormat finish() { } + private void fillSyncKerberosConfig( org.apache.hadoop.conf.Configuration config, String regionserverKeytabFile, String regionserverPrincipal, + String zookeeperSaslClient, String securityKrb5Conf) throws IOException { + if (StringUtils.isEmpty(regionserverKeytabFile)) { + throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); + } + String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; + LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); + + if (StringUtils.isEmpty(regionserverPrincipal)) { + throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); + } + config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); + config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); + + + if (!StringUtils.isEmpty(zookeeperSaslClient)) { + System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, zookeeperSaslClient); + } + + if (!StringUtils.isEmpty(securityKrb5Conf)) { + String krb5ConfPath = System.getProperty("user.dir") + File.separator + securityKrb5Conf; + LOG.info("krb5ConfPath:{}", krb5ConfPath); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); + } + } + } From a44c2be5167ad98b86d0036ec66e52db6c7cab7d Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 11:49:03 +0800 Subject: [PATCH 201/378] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 36b251d14..97c3ad873 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -92,8 +92,6 @@ public void configure(Configuration parameters) { @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); - conn = ConnectionFactory.createConnection(conf); - table = conn.getTable(TableName.valueOf(tableName)); if (kerberosAuthEnable) { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); @@ -117,6 +115,7 @@ public Connection run() { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); conn = ConnectionFactory.createConnection(conf); } + table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); } From 0de96e840c855eee6e52e71ca724a710dd497f85 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 27 May 2020 14:03:38 +0800 Subject: [PATCH 202/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9join?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E4=B8=AD=E9=9D=9Eequal=20=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AF=B9=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/MyLocalStreamEnvironment.java | 38 ++++-- .../flink/sql/side/JoinNodeDealer.java | 108 +++++++++++++++--- .../dtstack/flink/sql/util/TableUtils.java | 51 ++++++++- 3 files changed, 169 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 029a07f99..6aec8475e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -93,33 +94,48 @@ public JobExecutionResult execute(String jobName) throws Exception { // transform the streaming program into a JobGraph StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); + return execute(streamGraph); + } + + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.setClasspaths(classpaths); + jobGraph.setAllowQueuedScheduling(true); Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); + if (!configuration.contains(RestOptions.BIND_PORT)) { + configuration.setString(RestOptions.BIND_PORT, "0"); + } + + int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + + MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(configuration) + .setNumSlotsPerTaskManager(numSlotsPerTaskManager) + .build(); if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - try (MiniCluster exec = new MiniCluster(configBuilder.build());) { - exec.start(); - JobExecutionResult jobExecutionResult = exec.executeJobBlocking(jobGraph); + MiniCluster miniCluster = new MiniCluster(cfg); + + try { + miniCluster.start(); + configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); + + return miniCluster.executeJobBlocking(jobGraph); + } + finally { transformations.clear(); - return jobExecutionResult; - } catch (Exception e) { - throw new RuntimeException(e); + miniCluster.close(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index d2bb2cb0d..fd44dd47a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -512,10 +512,53 @@ private void extractJoinField(SqlNode condition, Set> joi } SqlKind joinKind = condition.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - extractJoinField(((SqlBasicCall)condition).operands[0], joinFieldSet); - extractJoinField(((SqlBasicCall)condition).operands[1], joinFieldSet); - }else{ + if ( AGGREGATE.contains(condition.getKind()) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + + SqlBasicCall sqlBasicCall = (SqlBasicCall) condition; + for(int i=0; i tuple2 = Tuple2.of(((SqlIdentifier)condition).names.get(0), ((SqlIdentifier)condition).names.get(1)); joinFieldSet.add(tuple2); @@ -822,20 +865,57 @@ public SqlBasicCall buildEmptyCondition(){ private SqlIdentifier checkAndReplaceJoinCondition(SqlNode node, Map tableMap){ SqlKind joinKind = node.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - SqlIdentifier leftNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[0], tableMap); - SqlIdentifier rightNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[1], tableMap); + if( AGGREGATE.contains(joinKind) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + SqlBasicCall sqlBasicCall = (SqlBasicCall) node; + for(int i=0; i Date: Wed, 27 May 2020 14:07:20 +0800 Subject: [PATCH 203/378] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=BC=BA=E5=A4=B1?= =?UTF-8?q?=E7=9A=84=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseConfigUtils.java | 14 +- .../sql/sink/hbase/HbaseOutputFormat.java | 132 ++++++++++++------ .../flink/sql/sink/hbase/HbaseSink.java | 9 ++ .../sql/sink/hbase/table/HbaseSinkParser.java | 7 + .../sql/sink/hbase/table/HbaseTableInfo.java | 20 +++ 5 files changed, 128 insertions(+), 54 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index 2be03484b..57c63d243 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -18,19 +18,12 @@ package com.dtstack.flink.sql.sink.hbase; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; /** * @@ -52,13 +45,18 @@ public class HbaseConfigUtils { public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "zookeeper.znode.parent"; + + public final static String KEY_HBASE_CLIENT_KEYTAB_FILE = "hbase.client.keytab.file"; + public final static String KEY_HBASE_CLIENT_KERBEROS_PRINCIPAL = "hbase.client.kerberos.principal"; public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + LOG.info("loginAndReturnUGI principal {}",principal); + LOG.info("loginAndReturnUGI keytab {}",keytab); if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 97c3ad873..3b17c3551 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -29,8 +29,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; @@ -70,6 +69,8 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private String regionserverPrincipal; private String securityKrb5Conf; private String zookeeperSaslClient; + private String clientPrincipal; + private String clientKeytabFile; private String[] families; private String[] qualifiers; @@ -78,48 +79,78 @@ public class HbaseOutputFormat extends AbstractDtRichOutputFormat { private transient Connection conn; private transient Table table; + private transient ChoreService choreService; + @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); conf = HBaseConfiguration.create(); - conf.set("hbase.zookeeper.quorum", host); - if (zkParent != null && !"".equals(zkParent)) { - conf.set("zookeeper.znode.parent", zkParent); - } - LOG.warn("---configure end ---"); } @Override public void open(int taskNumber, int numTasks) throws IOException { LOG.warn("---open---"); - if (kerberosAuthEnable) { - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - fillSyncKerberosConfig(conf, regionserverKeytabFile, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); - - UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, regionserverPrincipal, regionserverKeytabFile); - org.apache.hadoop.conf.Configuration finalConf = conf; - conn = userGroupInformation.doAs(new PrivilegedAction() { - @Override - public Connection run() { - try { - return ConnectionFactory.createConnection(finalConf); - } catch (IOException e) { - LOG.error("Get connection fail with config:{}", finalConf); - throw new RuntimeException(e); - } - } - }); - } else { - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); - conn = ConnectionFactory.createConnection(conf); - } + openConn(); table = conn.getTable(TableName.valueOf(tableName)); LOG.warn("---open end(get table from hbase) ---"); initMetric(); } + private void openConn(){ + try{ + if (kerberosAuthEnable) { + LOG.info("open kerberos conn"); + openKerberosConn(); + } else { + LOG.info("open conn"); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + conn = ConnectionFactory.createConnection(conf); + } + }catch (Exception e){ + throw new RuntimeException(e); + } + + } + private void openKerberosConn() throws IOException { + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); + + LOG.info("kerberos config:{}", this.toString()); + Preconditions.checkArgument(!StringUtils.isEmpty(clientPrincipal), " clientPrincipal not null!"); + Preconditions.checkArgument(!StringUtils.isEmpty(clientKeytabFile), " clientKeytabFile not null!"); + + fillSyncKerberosConfig(conf, regionserverPrincipal, zookeeperSaslClient, securityKrb5Conf); + + clientKeytabFile = System.getProperty("user.dir") + File.separator + clientKeytabFile; + clientPrincipal = !StringUtils.isEmpty(clientPrincipal) ? clientPrincipal : regionserverPrincipal; + + conf.set(HbaseConfigUtils.KEY_HBASE_CLIENT_KEYTAB_FILE, clientKeytabFile); + conf.set(HbaseConfigUtils.KEY_HBASE_CLIENT_KERBEROS_PRINCIPAL, clientPrincipal); + + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, clientPrincipal, clientKeytabFile); + org.apache.hadoop.conf.Configuration finalConf = conf; + conn = userGroupInformation.doAs(new PrivilegedAction() { + @Override + public Connection run() { + try { + ScheduledChore authChore = AuthUtil.getAuthChore(finalConf); + if (authChore != null) { + choreService = new ChoreService("hbaseKerberosSink"); + choreService.scheduleChore(authChore); + } + + return ConnectionFactory.createConnection(finalConf); + } catch (IOException e) { + LOG.error("Get connection fail with config:{}", finalConf); + throw new RuntimeException(e); + } + } + }); + } + + + @Override public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; @@ -227,7 +258,6 @@ public void close() throws IOException { conn = null; } } - private HbaseOutputFormat() { } @@ -264,11 +294,6 @@ public HbaseOutputFormatBuilder setRowkey(String rowkey) { return this; } - public HbaseOutputFormatBuilder setUpdateMode(String updateMode) { - format.updateMode = updateMode; - return this; - } - public HbaseOutputFormatBuilder setColumnNames(String[] columnNames) { format.columnNames = columnNames; return this; @@ -283,6 +308,7 @@ public HbaseOutputFormatBuilder setColumnNameFamily(Map columnNa format.columnNameFamily = columnNameFamily; return this; } + public HbaseOutputFormatBuilder setKerberosAuthEnable(boolean kerberosAuthEnable) { format.kerberosAuthEnable = kerberosAuthEnable; return this; @@ -308,6 +334,16 @@ public HbaseOutputFormatBuilder setZookeeperSaslClient(String zookeeperSaslClien return this; } + public HbaseOutputFormatBuilder setClientPrincipal(String clientPrincipal) { + format.clientPrincipal = clientPrincipal; + return this; + } + + public HbaseOutputFormatBuilder setClientKeytabFile(String clientKeytabFile) { + format.clientKeytabFile = clientKeytabFile; + return this; + } + public HbaseOutputFormat finish() { Preconditions.checkNotNull(format.host, "zookeeperQuorum should be specified"); @@ -323,7 +359,7 @@ public HbaseOutputFormat finish() { String[] columns = keySet.toArray(new String[keySet.size()]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; - String[] part = StringUtils.split(col, ":");; + String[] part = col.split(":"); families[i] = part[0]; qualifiers[i] = part[1]; } @@ -336,16 +372,8 @@ public HbaseOutputFormat finish() { } - private void fillSyncKerberosConfig( org.apache.hadoop.conf.Configuration config, String regionserverKeytabFile, String regionserverPrincipal, + private void fillSyncKerberosConfig(org.apache.hadoop.conf.Configuration config, String regionserverPrincipal, String zookeeperSaslClient, String securityKrb5Conf) throws IOException { - if (StringUtils.isEmpty(regionserverKeytabFile)) { - throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); - } - String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; - LOG.info("regionserverKeytabFilePath:{}",regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); - if (StringUtils.isEmpty(regionserverPrincipal)) { throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); } @@ -366,5 +394,17 @@ private void fillSyncKerberosConfig( org.apache.hadoop.conf.Configuration config } } + @Override + public String toString() { + return "HbaseOutputFormat kerberos{" + + "kerberosAuthEnable=" + kerberosAuthEnable + + ", regionserverKeytabFile='" + regionserverKeytabFile + '\'' + + ", regionserverPrincipal='" + regionserverPrincipal + '\'' + + ", securityKrb5Conf='" + securityKrb5Conf + '\'' + + ", zookeeperSaslClient='" + zookeeperSaslClient + '\'' + + ", clientPrincipal='" + clientPrincipal + '\'' + + ", clientKeytabFile='" + clientKeytabFile + '\'' + + '}'; + } } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java index 8364b6c10..09f5944b4 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseSink.java @@ -59,6 +59,9 @@ public class HbaseSink implements RetractStreamTableSink, IStreamSinkGener< protected String regionserverPrincipal; protected String securityKrb5Conf; protected String zookeeperSaslClient; + + private String clientPrincipal; + private String clientKeytabFile; private int parallelism = -1; @@ -83,6 +86,9 @@ public HbaseSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.securityKrb5Conf = hbaseTableInfo.getSecurityKrb5Conf(); this.zookeeperSaslClient = hbaseTableInfo.getZookeeperSaslClient(); + this.clientKeytabFile = hbaseTableInfo.getClientKeytabFile(); + this.clientPrincipal = hbaseTableInfo.getClientPrincipal(); + Integer tmpSinkParallelism = hbaseTableInfo.getParallelism(); if (tmpSinkParallelism != null) { this.parallelism = tmpSinkParallelism; @@ -104,6 +110,9 @@ public void emitDataStream(DataStream> dataStream) { builder.setSecurityKrb5Conf(securityKrb5Conf); builder.setZookeeperSaslClient(zookeeperSaslClient); + builder.setClientPrincipal(clientPrincipal); + builder.setClientKeytabFile(clientKeytabFile); + HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); dataStream.addSink(richSinkFunction); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 6374ddfaf..5105e0fc0 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -61,6 +61,10 @@ public class HbaseSinkParser extends AbstractTableParser { public static final String SECURITY_KRB5_CONF_KEY = "securityKrb5Conf"; public static final String ZOOKEEPER_SASL_CLINT_KEY = "zookeeperSaslClient"; + public static final String CLIENT_PRINCIPAL_KEY = "clientPrincipal"; + public static final String CLIENT_KEYTABFILE_KEY = "clientKeytabFile"; + + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -85,6 +89,9 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map hbaseConfig = Maps.newHashMap(); public HbaseTableInfo(){ @@ -212,4 +216,20 @@ public void setZookeeperSaslClient(String zookeeperSaslClient) { this.zookeeperSaslClient = zookeeperSaslClient; } + public String getClientPrincipal() { + return clientPrincipal; + } + + public void setClientPrincipal(String clientPrincipal) { + this.clientPrincipal = clientPrincipal; + } + + public String getClientKeytabFile() { + return clientKeytabFile; + } + + public void setClientKeytabFile(String clientKeytabFile) { + this.clientKeytabFile = clientKeytabFile; + } + } From 1f48d456f5fc236c9be7fcee85e20ea6f64d970b Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 14:16:25 +0800 Subject: [PATCH 204/378] =?UTF-8?q?=E5=BC=82=E6=AD=A5=E7=BB=B4=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/HbaseAsyncReqRow.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index f3a1d6691..26bf8ce28 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -98,6 +98,7 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; + Map hbaseConfig = hbaseSideTableInfo.getHbaseConfig(); ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, @@ -105,11 +106,18 @@ public void open(Configuration parameters) throws Exception { Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC, hbaseSideTableInfo.getParent()); - - if (hbaseSideTableInfo.isKerberosAuthEnable()) { - fillAsyncKerberosConfig(config, hbaseSideTableInfo); + config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); + HbaseConfigUtils.loadKrb5Conf(hbaseConfig); + hbaseConfig.entrySet().forEach(entity -> { + config.overrideConfig(entity.getKey(), (String) entity.getValue()); + }); + + if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); + String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); + config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); } + hBaseClient = new HBaseClient(config, executorService); try { From 46763a77dadbb2be3e783238730d529e3524e3fd Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 27 May 2020 14:39:28 +0800 Subject: [PATCH 205/378] =?UTF-8?q?hbase=20kerberos=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 50 +----- .../sql/side/hbase/HbaseAsyncReqRow.java | 27 --- .../sql/side/hbase/table/HbaseSideParser.java | 18 +- .../side/hbase/table/HbaseSideTableInfo.java | 11 ++ .../side/hbase/utils/HbaseConfigUtils.java | 167 +++++++++++++++--- 5 files changed, 161 insertions(+), 112 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index c50e1a806..376bccd81 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -179,22 +179,14 @@ private void loadData(Map> tmpCache) throws SQLExcep boolean openKerberos = hbaseSideTableInfo.isKerberosAuthEnable(); int loadDataCount = 0; try { - conf = HBaseConfiguration.create(); if (openKerberos) { + conf = HbaseConfigUtils.getHadoopConfiguration(hbaseSideTableInfo.getHbaseConfig()); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); - - fillSyncKerberosConfig(conf,hbaseSideTableInfo); - LOG.info("hbase.security.authentication:{}", conf.get("hbase.security.authentication")); - LOG.info("hbase.security.authorization:{}", conf.get("hbase.security.authorization")); - LOG.info("hbase.master.keytab.file:{}", conf.get("hbase.master.keytab.file")); - LOG.info("hbase.master.kerberos.principal:{}", conf.get("hbase.master.kerberos.principal")); - LOG.info("hbase.regionserver.keytab.file:{}", conf.get("hbase.regionserver.keytab.file")); - LOG.info("hbase.regionserver.kerberos.principal:{}", conf.get("hbase.regionserver.kerberos.principal")); - - UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, hbaseSideTableInfo.getRegionserverPrincipal(), - hbaseSideTableInfo.getRegionserverKeytabFile()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); + String principal = HbaseConfigUtils.getPrincipal(hbaseSideTableInfo.getHbaseConfig()); + String keytab = HbaseConfigUtils.getKeytab(hbaseSideTableInfo.getHbaseConfig()); + UserGroupInformation userGroupInformation = HbaseConfigUtils.loginAndReturnUGI(conf, principal, keytab); Configuration finalConf = conf; conn = userGroupInformation.doAs(new PrivilegedAction() { @Override @@ -209,8 +201,9 @@ public Connection run() { }); } else { + conf = HbaseConfigUtils.getConfig(hbaseSideTableInfo.getHbaseConfig()); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); - conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC, hbaseSideTableInfo.getParent()); + conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); conn = ConnectionFactory.createConnection(conf); } @@ -253,33 +246,4 @@ public Connection run() { } } - private void fillSyncKerberosConfig(Configuration config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { - String regionserverKeytabFile = hbaseSideTableInfo.getRegionserverKeytabFile(); - if (StringUtils.isEmpty(regionserverKeytabFile)) { - throw new IllegalArgumentException("Must provide regionserverKeytabFile when authentication is Kerberos"); - } - String regionserverKeytabFilePath = System.getProperty("user.dir") + File.separator + regionserverKeytabFile; - LOG.info("regionserverKeytabFilePath:{}", regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KEYTAB_FILE, regionserverKeytabFilePath); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KEYTAB_FILE, regionserverKeytabFilePath); - - String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); - if (StringUtils.isEmpty(regionserverPrincipal)) { - throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); - } - config.set(HbaseConfigUtils.KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, regionserverPrincipal); - config.set(HbaseConfigUtils.KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL, regionserverPrincipal); - config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHORIZATION, "true"); - config.set(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { - System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); - } - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { - String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); - LOG.info("krb5ConfPath:{}", krb5ConfPath); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); - } - } } \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 26bf8ce28..a6bfaca7a 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -187,33 +187,6 @@ public void close() throws Exception { hBaseClient.shutdown(); } - private void fillAsyncKerberosConfig(Config config, HbaseSideTableInfo hbaseSideTableInfo) throws IOException { - AuthUtil.JAASConfig jaasConfig = HbaseConfigUtils.buildJaasConfig(hbaseSideTableInfo); - LOG.info("jaasConfig file:\n {}", jaasConfig.toString()); - String jaasFilePath = AuthUtil.creatJaasFile("JAAS", ".conf", jaasConfig); - config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTH_ENABLE, "true"); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SASL_CLIENTCONFIG, "Client"); - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_SECURITY_AUTHENTICATION, "kerberos"); - - String regionserverPrincipal = hbaseSideTableInfo.getRegionserverPrincipal(); - if (StringUtils.isEmpty(regionserverPrincipal)) { - throw new IllegalArgumentException("Must provide regionserverPrincipal when authentication is Kerberos"); - } - config.overrideConfig(HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, regionserverPrincipal); - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getZookeeperSaslClient())) { - System.setProperty(HbaseConfigUtils.KEY_ZOOKEEPER_SASL_CLIENT, hbaseSideTableInfo.getZookeeperSaslClient()); - } - - if (!StringUtils.isEmpty(hbaseSideTableInfo.getSecurityKrb5Conf())) { - String krb5ConfPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getSecurityKrb5Conf(); - LOG.info("krb5ConfPath:{}", krb5ConfPath); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_KRB5_CONF, krb5ConfPath); - } - } - - class CheckResult{ private boolean connect; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java index 832961834..03868d618 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideParser.java @@ -53,13 +53,6 @@ public class HbaseSideParser extends AbstractSideTableParser { public static final String CACHE = "cache"; - public static final String KERBEROS_AUTH_ENABLE_KEY = "kerberosAuthEnable"; - public static final String REGIONSERVER_KEYTAB_FILE_KEY = "regionserverKeytabFile"; - public static final String REGIONSERVER_PRINCIPAL_KEY = "regionserverPrincipal"; - public static final String JAAS_PRINCIPAL_KEY = "jaasPrincipal"; - public static final String SECURITY_KRB5_CONF_KEY = "securityKrb5Conf"; - public static final String ZOOKEEPER_SASL_CLINT_KEY = "zookeeperSaslClient"; - public HbaseSideParser() { addParserHandler(FIELD_KEY, FIELD_PATTERN, this::dealField); } @@ -76,13 +69,10 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map entity.getKey().contains(".")) + .map(entity -> hbaseTableInfo.getHbaseConfig().put(entity.getKey(), String.valueOf(entity.getValue()))) + .count(); return hbaseTableInfo; } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java index 2a5411682..51597d583 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/table/HbaseSideTableInfo.java @@ -50,6 +50,8 @@ public class HbaseSideTableInfo extends AbstractSideTableInfo { private Map columnNameFamily; + private Map hbaseConfig = Maps.newHashMap(); + private String tableName; private boolean kerberosAuthEnable; @@ -208,6 +210,15 @@ public void setZookeeperSaslClient(String zookeeperSaslClient) { this.zookeeperSaslClient = zookeeperSaslClient; } + public Map getHbaseConfig() { + return hbaseConfig; + } + + public void setHbaseConfig(Map hbaseConfig) { + this.hbaseConfig = hbaseConfig; + } + + @Override public void finish(){ super.finish(); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index 09f2ea53d..a7708aaae 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -18,8 +18,6 @@ package com.dtstack.flink.sql.side.hbase.utils; -import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; -import com.dtstack.flink.sql.util.AuthUtil; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -33,7 +31,6 @@ import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -50,42 +47,157 @@ public class HbaseConfigUtils { private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigUtils.class); // sync side kerberos - public final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; - public final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; - public final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; - public final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; - public final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; - public final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; + private final static String AUTHENTICATION_TYPE = "Kerberos"; + private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication"; + private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization"; + private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; + private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file"; + private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; // async side kerberos - public final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; - public final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; - public final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; + private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; + private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + private static final String KEY_KEY_TAB = "hbase.keytab"; + private static final String KEY_PRINCIPAL = "hbase.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_SYNC = "zookeeper.znode.parent"; - public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM_ASYNC = "hbase.zookeeper.znode.parent"; + public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; - public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; - public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; - + private static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_JAVA_SECURITY_AUTH_LOGIN_CONF = "java.security.auth.login.config"; - public static AuthUtil.JAASConfig buildJaasConfig(HbaseSideTableInfo hbaseSideTableInfo) { - String keytabPath = System.getProperty("user.dir") + File.separator + hbaseSideTableInfo.getRegionserverKeytabFile(); - Map loginModuleOptions = new HashMap<>(); - loginModuleOptions.put("useKeyTab", "true"); - loginModuleOptions.put("useTicketCache", "false"); - loginModuleOptions.put("keyTab", "\"" + keytabPath + "\""); - loginModuleOptions.put("principal", "\"" + hbaseSideTableInfo.getJaasPrincipal() + "\""); - return AuthUtil.JAASConfig.builder().setEntryName("Client") - .setLoginModule("com.sun.security.auth.module.Krb5LoginModule") - .setLoginModuleFlag("required").setLoginModuleOptions(loginModuleOptions).build(); + private static final String SP = File.separator; + private static final String KEY_KRB5_CONF = "krb5.conf"; + + + private static List KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_HBASE_MASTER_KERBEROS_PRINCIPAL, + KEY_HBASE_MASTER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KEYTAB_FILE, + KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL + ); + + private static List ASYNC_KEYS_KERBEROS_REQUIRED = Arrays.asList( + KEY_HBASE_SECURITY_AUTH_ENABLE, + KEY_HBASE_SASL_CLIENTCONFIG, + KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL, + KEY_HBASE_SECURITY_AUTHENTICATION, + KEY_KEY_TAB); + + + public static Configuration getConfig(Map hbaseConfigMap) { + Configuration hConfiguration = HBaseConfiguration.create(); + + for (Map.Entry entry : hbaseConfigMap.entrySet()) { + if (entry.getValue() != null && !(entry.getValue() instanceof Map)) { + hConfiguration.set(entry.getKey(), entry.getValue().toString()); + } + } + return hConfiguration; + } + + public static boolean openKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + } + + public static boolean asyncOpenKerberos(Map hbaseConfigMap) { + if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { + return false; + } + return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); } + + + public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { + for (String key : KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + loadKrb5Conf(hbaseConfigMap); + + Configuration conf = new Configuration(); + if (hbaseConfigMap == null) { + return conf; + } + + hbaseConfigMap.forEach((key, val) -> { + if (val != null) { + conf.set(key, val.toString()); + } + }); + + return conf; + } + + public static String getPrincipal(Map hbaseConfigMap) { + String principal = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KERBEROS_PRINCIPAL); + if (StringUtils.isNotEmpty(principal)) { + return principal; + } + + throw new IllegalArgumentException(""); + } + + public static String getKeytab(Map hbaseConfigMap) { + String keytab = MapUtils.getString(hbaseConfigMap, KEY_HBASE_MASTER_KEYTAB_FILE); + if (StringUtils.isNotEmpty(keytab)) { + return keytab; + } + + throw new IllegalArgumentException(""); + } + + public static void loadKrb5Conf(Map kerberosConfig) { + String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; + } + } + + public static String creatJassFile(String configStr) throws IOException { + String fileName = System.getProperty("user.dir"); + File krbConf = new File(fileName); + File temp = File.createTempFile("JAAS", ".conf", krbConf); + temp.deleteOnExit(); + BufferedWriter out = new BufferedWriter(new FileWriter(temp, false)); + out.write(configStr + "\n"); + out.close(); + return temp.getAbsolutePath(); + } + + public static String buildJaasStr(Map kerberosConfig) { + for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { + if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { + throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); + } + } + + String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); + String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); + + StringBuilder jaasSB = new StringBuilder("Client {\n" + + " com.sun.security.auth.module.Krb5LoginModule required\n" + + " useKeyTab=true\n" + + " useTicketCache=false\n"); + jaasSB.append(" keyTab=\"").append(keyTab).append("\"").append("\n"); + jaasSB.append(" principal=\"").append(principal).append("\"").append(";\n"); + jaasSB.append("};"); + return jaasSB.toString(); + } + + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); @@ -104,5 +216,4 @@ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); } - } From fa0f6f2535e3bebb31c523115f31d6b29fe25a21 Mon Sep 17 00:00:00 2001 From: xuchao Date: Wed, 27 May 2020 15:12:06 +0800 Subject: [PATCH 206/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0launch=20main=20?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E7=9A=84=E6=97=A5=E5=BF=97=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 6 ++++++ .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 8 ++++++++ launcher/src/main/resource/log4j.properties | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 launcher/src/main/resource/log4j.properties diff --git a/launcher/pom.xml b/launcher/pom.xml index 1771df1ec..ea921d87a 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -29,6 +29,12 @@ 1.2.17 + + org.slf4j + slf4j-log4j12 + 1.6.1 + + com.alibaba fastjson diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 551a00533..b0b96332a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -41,6 +41,8 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -57,6 +59,8 @@ */ public class LauncherMain { + + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); private static final String CORE_JAR = "core"; private static String SP = File.separator; @@ -68,9 +72,13 @@ private static String getLocalCoreJarPath(String localSqlRootJar) throws Excepti } public static void main(String[] args) throws Exception { + + LOG.info("----start----"); + if (args.length == 1 && args[0].endsWith(".json")){ args = parseJson(args); } + OptionParser optionParser = new OptionParser(args); Options launcherOptions = optionParser.getOptions(); String mode = launcherOptions.getMode(); diff --git a/launcher/src/main/resource/log4j.properties b/launcher/src/main/resource/log4j.properties new file mode 100644 index 000000000..44e3a5ef2 --- /dev/null +++ b/launcher/src/main/resource/log4j.properties @@ -0,0 +1,7 @@ +#控制台输出： +log4j.rootLogger = INFO,consoleAppender +log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender +log4j.appender.console.Target = System.out +log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout +log4j.appender.consoleAppender.layout.ConversionPattern =%d %-5p %m %n +log4j.appender.consoleAppender.ImmediateFlush = true \ No newline at end of file From ea3ba9d46f27ca669dd7f6a1fd5fe7dcb0f7457a Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 21 May 2020 20:00:40 +0800 Subject: [PATCH 207/378] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E4=B9=B1=E5=90=97=E9=97=AE=E9=A2=98...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 16 ++++++++++++++-- .../flink/sql/sink/impala/ImpalaSink.java | 4 ++-- .../sql/sink/impala/table/ImpalaSinkParser.java | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 3eaffc0ad..89fb769c3 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.typeinfo.TypeInformation; import java.util.Arrays; import java.util.List; @@ -38,6 +39,12 @@ public class ImpalaDialect implements JDBCDialect { private static final String IMPALA_PARTITION_KEYWORD = "partition"; + private TypeInformation[] fieldTypes; + + public ImpalaDialect(TypeInformation[] fieldTypes){ + this.fieldTypes = fieldTypes; + } + @Override public boolean canHandle(String url) { return url.startsWith("jdbc:impala:"); @@ -70,8 +77,13 @@ public String getInsertIntoStatement(String schema, String tableName, String[] f .map(this::quoteIdentifier) .collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fieldNames) - .map(f -> "?") + String placeholders = Arrays.stream(fieldTypes) + .map(f -> { + if(String.class.getName().equals(f.getTypeClass().getName())){ + return "cast( ? as string)"; + } + return "?"; + }) .collect(Collectors.joining(", ")); String partitionFieldStr = partitionFieldsList.stream() diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index cd1e1e945..910f6f4c5 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -41,14 +41,14 @@ public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener Date: Wed, 27 May 2020 14:03:38 +0800 Subject: [PATCH 208/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9join?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E4=B8=AD=E9=9D=9Eequal=20=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AF=B9=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/MyLocalStreamEnvironment.java | 38 ++++-- .../flink/sql/side/JoinNodeDealer.java | 108 +++++++++++++++--- .../dtstack/flink/sql/util/TableUtils.java | 51 ++++++++- 3 files changed, 169 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java index 029a07f99..6aec8475e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -93,33 +94,48 @@ public JobExecutionResult execute(String jobName) throws Exception { // transform the streaming program into a JobGraph StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); + return execute(streamGraph); + } + + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.setClasspaths(classpaths); + jobGraph.setAllowQueuedScheduling(true); Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); + if (!configuration.contains(RestOptions.BIND_PORT)) { + configuration.setString(RestOptions.BIND_PORT, "0"); + } + + int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + + MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(configuration) + .setNumSlotsPerTaskManager(numSlotsPerTaskManager) + .build(); if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - try (MiniCluster exec = new MiniCluster(configBuilder.build());) { - exec.start(); - JobExecutionResult jobExecutionResult = exec.executeJobBlocking(jobGraph); + MiniCluster miniCluster = new MiniCluster(cfg); + + try { + miniCluster.start(); + configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); + + return miniCluster.executeJobBlocking(jobGraph); + } + finally { transformations.clear(); - return jobExecutionResult; - } catch (Exception e) { - throw new RuntimeException(e); + miniCluster.close(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index d2bb2cb0d..fd44dd47a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -512,10 +512,53 @@ private void extractJoinField(SqlNode condition, Set> joi } SqlKind joinKind = condition.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - extractJoinField(((SqlBasicCall)condition).operands[0], joinFieldSet); - extractJoinField(((SqlBasicCall)condition).operands[1], joinFieldSet); - }else{ + if ( AGGREGATE.contains(condition.getKind()) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + + SqlBasicCall sqlBasicCall = (SqlBasicCall) condition; + for(int i=0; i tuple2 = Tuple2.of(((SqlIdentifier)condition).names.get(0), ((SqlIdentifier)condition).names.get(1)); joinFieldSet.add(tuple2); @@ -822,20 +865,57 @@ public SqlBasicCall buildEmptyCondition(){ private SqlIdentifier checkAndReplaceJoinCondition(SqlNode node, Map tableMap){ SqlKind joinKind = node.getKind(); - if( joinKind == AND || joinKind == EQUALS ){ - SqlIdentifier leftNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[0], tableMap); - SqlIdentifier rightNode = checkAndReplaceJoinCondition(((SqlBasicCall)node).operands[1], tableMap); + if( AGGREGATE.contains(joinKind) + || AVG_AGG_FUNCTIONS.contains(joinKind) + || COMPARISON.contains(joinKind) + || joinKind == OTHER_FUNCTION + || joinKind == DIVIDE + || joinKind == CAST + || joinKind == TRIM + || joinKind == TIMES + || joinKind == PLUS + || joinKind == NOT_IN + || joinKind == OR + || joinKind == AND + || joinKind == MINUS + || joinKind == TUMBLE + || joinKind == TUMBLE_START + || joinKind == TUMBLE_END + || joinKind == SESSION + || joinKind == SESSION_START + || joinKind == SESSION_END + || joinKind == HOP + || joinKind == HOP_START + || joinKind == HOP_END + || joinKind == BETWEEN + || joinKind == IS_NULL + || joinKind == IS_NOT_NULL + || joinKind == CONTAINS + || joinKind == TIMESTAMP_ADD + || joinKind == TIMESTAMP_DIFF + || joinKind == LIKE + || joinKind == COALESCE + || joinKind == EQUALS ){ + SqlBasicCall sqlBasicCall = (SqlBasicCall) node; + for(int i=0; i Date: Wed, 27 May 2020 15:12:06 +0800 Subject: [PATCH 209/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0launch=20main=20?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E7=9A=84=E6=97=A5=E5=BF=97=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 6 ++++++ .../java/com/dtstack/flink/sql/launcher/LauncherMain.java | 8 ++++++++ launcher/src/main/resource/log4j.properties | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 launcher/src/main/resource/log4j.properties diff --git a/launcher/pom.xml b/launcher/pom.xml index 1771df1ec..ea921d87a 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -29,6 +29,12 @@ 1.2.17 + + org.slf4j + slf4j-log4j12 + 1.6.1 + + com.alibaba fastjson diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 551a00533..b0b96332a 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -41,6 +41,8 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -57,6 +59,8 @@ */ public class LauncherMain { + + private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); private static final String CORE_JAR = "core"; private static String SP = File.separator; @@ -68,9 +72,13 @@ private static String getLocalCoreJarPath(String localSqlRootJar) throws Excepti } public static void main(String[] args) throws Exception { + + LOG.info("----start----"); + if (args.length == 1 && args[0].endsWith(".json")){ args = parseJson(args); } + OptionParser optionParser = new OptionParser(args); Options launcherOptions = optionParser.getOptions(); String mode = launcherOptions.getMode(); diff --git a/launcher/src/main/resource/log4j.properties b/launcher/src/main/resource/log4j.properties new file mode 100644 index 000000000..44e3a5ef2 --- /dev/null +++ b/launcher/src/main/resource/log4j.properties @@ -0,0 +1,7 @@ +#控制台输出： +log4j.rootLogger = INFO,consoleAppender +log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender +log4j.appender.console.Target = System.out +log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout +log4j.appender.consoleAppender.layout.ConversionPattern =%d %-5p %m %n +log4j.appender.consoleAppender.ImmediateFlush = true \ No newline at end of file From eb0ae659caf4fef618262dcf96c6999e3123281c Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 28 May 2020 15:00:19 +0800 Subject: [PATCH 210/378] =?UTF-8?q?fix=20=E7=BA=BF=E7=A8=8B=E6=95=B0?= =?UTF-8?q?=E5=92=8C=E9=98=9F=E5=88=97=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java | 4 ++-- .../java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java | 5 +++-- .../com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java | 4 ++-- .../com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java | 4 ++-- .../com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java | 4 ++-- .../dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java | 4 ++-- .../flink/sql/side/postgresql/PostgresqlAsyncReqRow.java | 4 ++-- .../flink/sql/side/sqlserver/SqlserverAsyncReqRow.java | 4 ++-- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index 742f58d68..d251bf98b 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -67,8 +67,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("clickhouseAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("clickhouseAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index dc55bf604..e7ae0b650 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -76,8 +76,9 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("dbAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("dbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); + } } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 3f2172f6d..51fe6abc5 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -83,8 +83,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("impalaAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("impalaAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 4ac7e8ba5..ffdd09b51 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -76,8 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("mysqlAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("mysqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index 53d6ff896..b60a794b1 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -68,7 +68,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("oracleAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("oracleAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 63b80ef0d..333b9e7e2 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -75,7 +75,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("polardbAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("polardbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index cedcb9eca..3bc745e4c 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -76,8 +76,8 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("postgresqlAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("postgresqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index e8a22dfe8..aadf2b811 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -78,7 +78,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); - setExecutor(new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10), new DTThreadFactory("sqlServerAsyncExec"))); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("sqlServerAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 814584977637ab8c2a8d2e9863e440de06c8f030 Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 28 May 2020 17:04:50 +0800 Subject: [PATCH 211/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B9=8B=E5=89=8D?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E4=BB=A3=E7=A0=81=E5=86=B2=E7=AA=81=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E4=BB=A3=E7=A0=81=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/kafka/AbstractKafkaProducerFactory.java | 3 --- .../DTJsonRowSerializationSchema.java | 3 ++- .../JsonCRowSerializationSchema.java | 16 ++++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) rename kafka-base/kafka-base-sink/src/main/java/{org/apache/flink/formats/json => com/dtstack/flink/sql/sink/kafka/serialization}/DTJsonRowSerializationSchema.java (98%) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java index 429b260b1..9958a2544 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaProducerFactory.java @@ -26,9 +26,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.formats.avro.AvroRowSerializationSchema; -import org.apache.flink.formats.csv.CsvRowSerializationSchema; -import org.apache.flink.formats.json.DTJsonRowSerializationSchema; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.table.runtime.types.CRow; diff --git a/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/DTJsonRowSerializationSchema.java similarity index 98% rename from kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java rename to kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/DTJsonRowSerializationSchema.java index f67d2353d..da22dcc55 100644 --- a/kafka-base/kafka-base-sink/src/main/java/org/apache/flink/formats/json/DTJsonRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/DTJsonRowSerializationSchema.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.flink.formats.json; +package com.dtstack.flink.sql.sink.kafka.serialization; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.serialization.SerializationSchema; @@ -25,6 +25,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.formats.json.JsonRowSchemaConverter; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java index bfe801d52..34491d367 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/serialization/JsonCRowSerializationSchema.java @@ -159,34 +159,34 @@ private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { } private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { - if (info == Types.VOID || object == null) { + if (info.equals(Types.VOID) || object == null) { return container.nullNode(); - } else if (info == Types.BOOLEAN) { + } else if (info.equals(Types.BOOLEAN)) { return container.booleanNode((Boolean) object); - } else if (info == Types.STRING) { + } else if (info.equals(Types.STRING)) { return container.textNode((String) object); - } else if (info == Types.BIG_DEC) { + } else if (info.equals(Types.BIG_DEC)) { // convert decimal if necessary if (object instanceof BigDecimal) { return container.numberNode((BigDecimal) object); } return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); - } else if (info == Types.BIG_INT) { + } else if (info.equals(Types.BIG_INT)) { // convert integer if necessary if (object instanceof BigInteger) { return container.numberNode((BigInteger) object); } return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); - } else if (info == Types.SQL_DATE) { + } else if (info.equals(Types.SQL_DATE)) { return container.textNode(object.toString()); - } else if (info == Types.SQL_TIME) { + } else if (info.equals(Types.SQL_TIME)) { final Time time = (Time) object; // strip milliseconds if possible if (time.getTime() % 1000 > 0) { return container.textNode(timeFormatWithMillis.format(time)); } return container.textNode(timeFormat.format(time)); - } else if (info == Types.SQL_TIMESTAMP) { + } else if (info.equals(Types.SQL_TIMESTAMP)) { return container.textNode(timestampFormat.format((Timestamp) object)); } else if (info instanceof RowTypeInfo) { if (reuse != null && reuse instanceof ObjectNode) { From 44be6d0bcc44e160c422dbacf6ac2cb1f5e9a587 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 12:43:58 +0800 Subject: [PATCH 212/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=A9=BA=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 3b17c3551..3f18a3a20 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -165,7 +165,7 @@ public void writeRecord(Tuple2 tuple2) { protected void dealInsert(Row record) { Put put = getPutByRow(record); - if (put == null) { + if (put.isEmpty()) { return; } From 28ae8889c12cda67769fe59be5e657d8cc364af3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 14:05:59 +0800 Subject: [PATCH 213/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=A9=BA=E7=BD=AE?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 3f18a3a20..acb6b5448 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -165,7 +165,7 @@ public void writeRecord(Tuple2 tuple2) { protected void dealInsert(Row record) { Put put = getPutByRow(record); - if (put.isEmpty()) { + if (put == null || put.isEmpty()) { return; } From 9371b9d6244c74c00932df023375225ba2a568df Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 14:32:30 +0800 Subject: [PATCH 214/378] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E8=84=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index acb6b5448..a6a889cc4 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -166,12 +166,13 @@ public void writeRecord(Tuple2 tuple2) { protected void dealInsert(Row record) { Put put = getPutByRow(record); if (put == null || put.isEmpty()) { + outDirtyRecords.inc(); return; } try { table.put(put); - } catch (IOException e) { + } catch (Exception e) { if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ..{}", record.toString()); LOG.error("", e); From baafc80800913455d35cb80aa5c6b32d4f005af7 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 3 Jun 2020 20:13:39 +0800 Subject: [PATCH 215/378] =?UTF-8?q?kerberos=20=E8=AF=AD=E5=8F=A5=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/config.md | 5 +++ docs/plugin/hbaseSide.md | 80 +++++++++++++++++++++++++++++++++++++++- docs/plugin/hbaseSink.md | 64 ++++++++++++++++++++++++++++++-- 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/docs/config.md b/docs/config.md index 9aa8df994..27c4244e7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -46,6 +46,11 @@ sh submit.sh -key1 val1 -key2 val2 * 描述：扩展jar路径,当前主要是UDF定义的jar； * 必选：否 * 默认值：无 + +* **addShipfile** + * 描述：扩展上传的文件，比如开启；Kerberos认证需要的keytab文件和krb5.conf文件 + * 必选：否 + * 默认值：无 * **confProp** * 描述：一些参数设置 diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index e590b02e7..29dc60bf9 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -43,7 +43,14 @@ | tableName | hbase 的表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - +|kerberosAuthEnable | 是否开启kerberos认证|否|false| +|regionserverPrincipal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| +|clientKeytabFile|client的keytab 文件|否| +|clientPrincipal|client的principal|否|| +|zookeeperSaslClient | zookeeper.sasl.client值|否|true| +|securityKrb5Conf | java.security.krb5.conf值|否|| + 另外开启Kerberos认证还需要在VM参数中配置krb5, -Djava.security.krb5.conf=/Users/xuchao/Documents/flinkSql/kerberos/krb5.conf + 同时在addShipfile参数中添加keytab文件的路径，参数具体细节请看[命令参数说明](../config.md) -------------- ## 5.样例 @@ -168,4 +175,75 @@ into sideTable b on a.id=b.rowkey1 and a.name = b.rowkey2; ``` +### kerberos维表示例 +``` +CREATE TABLE MyTable( + name varchar, + channel varchar, + pv INT, + xctime bigint +)WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' +); + +CREATE TABLE MyResult( + name varchar, + channel varchar +)WITH( + type ='mysql', + url ='jdbc:mysql://172.16.10.45:3306/test', + userName ='dtstack', + password ='abc123', + tableName ='myresult', + updateMode ='append', + parallelism ='1', + batchSize ='100', + batchWaitInterval ='1000' +); + +CREATE TABLE sideTable( + cf:name varchar as name, + cf:info varchar as info, + PRIMARY KEY(md5(name) +'test') , + PERIOD FOR SYSTEM_TIME +)WITH( + type ='hbase', + zookeeperQuorum ='172.16.10.104:2181,172.16.10.224:2181,172.16.10.252:2181', + zookeeperParent ='/hbase', + tableName ='workerinfo', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + asyncTimeoutNum ='0', + parallelism ='1', + kerberosAuthEnable='true', + regionserverPrincipal='hbase/_HOST@DTSTACK.COM', + clientKeytabFile='test.keytab', + clientPrincipal='test@DTSTACK.COM', + securityKrb5Conf='krb5.conf', +); +insert into + MyResult +select + b.name as name, + a.channel + +from + MyTable a + +join + sideTable b + +on a.channel=b.name +``` diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index ef1be339b..5006f11a2 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -37,9 +37,17 @@ hbase2.0 |rowkey | hbase的rowkey关联的列信息,多个值以逗号隔开|是|| |updateMode|APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|APPEND｜ |parallelism | 并行度设置|否|1| - - +|kerberosAuthEnable | 是否开启kerberos认证|否|false| +|regionserverPrincipal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| +|clientKeytabFile|client的keytab 文件|否| +|clientPrincipal|client的principal|否|| +|zookeeperSaslClient | zookeeper.sasl.client值|否|true| +|securityKrb5Conf | java.security.krb5.conf值|否|| + 另外开启Kerberos认证还需要在VM参数中配置krb5, -Djava.security.krb5.conf=/Users/xuchao/Documents/flinkSql/kerberos/krb5.conf + 同时在addShipfile参数中添加keytab文件的路径，参数具体细节请看[命令参数说明](../config.md) ## 5.样例： + +### 普通结果表语句示例 ``` CREATE TABLE MyTable( name varchar, @@ -78,9 +86,59 @@ into channel, name from - MyTable a + MyTable a + + ``` +### kerberos认证结果表语句示例 +``` +CREATE TABLE MyTable( + name varchar, + channel varchar, + age int + )WITH( + type ='kafka10', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='mqTest01', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + +CREATE TABLE MyResult( + cf:name varchar , + cf:channel varchar + )WITH( + type ='hbase', + zookeeperQuorum ='cdh2.cdhsite:2181,cdh4.cdhsite:2181', + zookeeperParent ='/hbase', + tableName ='myresult', + partitionedJoin ='false', + parallelism ='1', + rowKey='name', + kerberosAuthEnable='true', + regionserverPrincipal='hbase/_HOST@DTSTACK.COM', + clientKeytabFile='test.keytab', + clientPrincipal='test@DTSTACK.COM', + securityKrb5Conf='krb5.conf', + ); + +insert +into + MyResult + select + channel, + name + from + MyTable a + +``` + ## 6.hbase数据 ### 数据内容说明 hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'-'连接 From 75f03491eea0f68a92e62ae2bbf1698e20c69f24 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 14:14:07 +0800 Subject: [PATCH 216/378] =?UTF-8?q?fix=20=E5=8F=82=E6=95=B0=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/CreateTableParser.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index ae6e1f708..9e3a96cec 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -41,6 +41,8 @@ public class CreateTableParser implements IParser { private static final Pattern PATTERN = Pattern.compile(PATTERN_STR); + private static final Pattern PROP_PATTERN = Pattern.compile("^'\\s*(.+)\\s*'$"); + public static CreateTableParser newInstance(){ return new CreateTableParser(); } @@ -69,18 +71,27 @@ public void parseSql(String sql, SqlTree sqlTree) { } private Map parseProp(String propsStr){ - String[] strs = propsStr.trim().split("'\\s*,"); + propsStr = propsStr.replaceAll("'\\s*,", "'|"); + String[] strs = propsStr.trim().split("\\|"); Map propMap = Maps.newHashMap(); for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); String key = ss.get(0).trim(); - String value = ss.get(1).trim().replaceAll("'", "").trim(); + String value = extractValue(ss.get(1)); propMap.put(key, value); } return propMap; } + private String extractValue(String value) { + Matcher matcher = PROP_PATTERN.matcher(value); + if (matcher.find()) { + return matcher.group(1); + } + throw new RuntimeException("[" + value + "] format is invalid"); + } + public static class SqlParserResult{ private String tableName; From 715bce35607eb0354a7875f90fbcbdd42453eadf Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 15:42:44 +0800 Subject: [PATCH 217/378] fix compile fail --- core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index 2320fddf3..bb8faf8ee 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -114,7 +114,7 @@ public void exec(String sql, SideSQLParser sideSQLParser = new SideSQLParser(); sideSQLParser.setLocalTableCache(localTableCache); - Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet()); + Queue exeQueue = sideSQLParser.getExeQueue(sql, sideTableMap.keySet(), scope); Object pollObj; //need clean From 46c7f3c8ec1b583a6629b40749602a031420f428 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 15:45:49 +0800 Subject: [PATCH 218/378] fix compile fail --- .../main/java/com/dtstack/flink/sql/launcher/LauncherMain.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index f8d063cce..27f1c64c9 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -63,8 +63,6 @@ public class LauncherMain { private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); private static final String CORE_JAR = "core"; - private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class); - private static String SP = File.separator; private static String getLocalCoreJarPath(String localSqlRootJar) throws Exception { From a8f16d6cb784b9d760508ab3880959951a849e1b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 17:40:38 +0800 Subject: [PATCH 219/378] remove blank --- .../java/com/dtstack/flink/sql/parser/CreateTableParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 9e3a96cec..91b1fd2ac 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -77,7 +77,7 @@ private Map parseProp(String propsStr){ for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); String key = ss.get(0).trim(); - String value = extractValue(ss.get(1)); + String value = extractValue(ss.get(1).trim()); propMap.put(key, value); } From a923fff36b088b48cb79424962f4e3c153d3ba3b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 21:08:11 +0800 Subject: [PATCH 220/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0addshipfile=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/option/Options.java | 13 +++++++++++++ .../launcher/perjob/PerJobClusterClientBuilder.java | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/core/src/main/java/com/dtstack/flink/sql/option/Options.java index ba6296d1e..b7d425a53 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/core/src/main/java/com/dtstack/flink/sql/option/Options.java @@ -72,6 +72,10 @@ public class Options { @OptionRequired(description = "log level") private String logLevel = "info"; + @OptionRequired(description = "file add to ship file") + private String addShipfile; + + public String getMode() { return mode; } @@ -183,4 +187,13 @@ public String getLogLevel() { public void setLogLevel(String logLevel) { this.logLevel = logLevel; } + + public String getAddShipfile() { + return addShipfile; + } + + public void setAddShipfile(String addShipfile) { + this.addShipfile = addShipfile; + } + } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 5dc74900e..9d99f2928 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -106,6 +106,13 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ } else { throw new RuntimeException("The Flink jar path is null"); } + // add user customized file to shipfile + if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { + List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); + paths.forEach(path -> { + shipFiles.add(new File(path)); + }); + } // classpath , all node need contain plugin jar String pluginLoadMode = launcherOptions.getPluginLoadMode(); if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { From 4b462d842256fcb49e99c31f1e183715a2ba2537 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 4 Jun 2020 21:10:26 +0800 Subject: [PATCH 221/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=BC=BA=E5=A4=B1?= =?UTF-8?q?=E7=9A=84=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/launcher/perjob/ConfigParseUtil.java | 32 ++++++++----------- .../perjob/PerJobClusterClientBuilder.java | 19 ++++++----- 2 files changed, 25 insertions(+), 26 deletions(-) rename clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java => launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java (55%) diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java similarity index 55% rename from clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java rename to launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java index 29bf54798..5acdb59a1 100644 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/ConfigParseUtil.java @@ -16,31 +16,27 @@ * limitations under the License. */ -package com.dtstack.flink.sql.sink.clickhouse; +package com.dtstack.flink.sql.launcher.perjob; -import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; +import org.apache.commons.io.Charsets; -import java.util.Optional; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; /** - * Date: 2020/1/15 + * Date: 2019/12/28 * Company: www.dtstack.com * @author maqi */ -public class ClickhouseDialect implements JDBCDialect { +public class ConfigParseUtil { - @Override - public boolean canHandle(String url) { - return url.startsWith("jdbc:clickhouse:"); - } - - @Override - public Optional defaultDriverName() { - return Optional.of("ru.yandex.clickhouse.ClickHouseDriver"); - } - - @Override - public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { - throw new RuntimeException("Clickhouse does not support update sql, please remove primary key or use append mode"); + public static List parsePathFromStr(String pathStr) throws UnsupportedEncodingException { + String addjarPath = URLDecoder.decode(pathStr, Charsets.UTF_8.toString()); + if (addjarPath.length() > 2) { + addjarPath = addjarPath.substring(1,addjarPath.length()-1).replace("\"",""); + } + return Arrays.asList(addjarPath.split(",")); } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 9d99f2928..84a4e23a3 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; @@ -82,7 +83,7 @@ public void init(String yarnConfDir, Configuration flinkConfig, Properties userC } public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJarPath, Options launcherOptions, JobGraph jobGraph) - throws MalformedURLException { + throws MalformedURLException, UnsupportedEncodingException { String flinkConf = StringUtils.isEmpty(launcherOptions.getFlinkconf()) ? DEFAULT_CONF_DIR : launcherOptions.getFlinkconf(); AbstractYarnClusterDescriptor clusterDescriptor = getClusterDescriptor(flinkConfig, yarnConf, flinkConf); @@ -106,13 +107,7 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ } else { throw new RuntimeException("The Flink jar path is null"); } - // add user customized file to shipfile - if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { - List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); - paths.forEach(path -> { - shipFiles.add(new File(path)); - }); - } + // classpath , all node need contain plugin jar String pluginLoadMode = launcherOptions.getPluginLoadMode(); if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { @@ -125,6 +120,14 @@ public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(String flinkJ + " Currently only classpath and shipfile are supported."); } + // add user customized file to shipfile + if (!StringUtils.isBlank(launcherOptions.getAddShipfile())) { + List paths = ConfigParseUtil.parsePathFromStr(launcherOptions.getAddShipfile()); + paths.forEach(path -> { + shipFiles.add(new File(path)); + }); + } + clusterDescriptor.addShipFiles(shipFiles); clusterDescriptor.setName(launcherOptions.getName()); String queue = launcherOptions.getQueue(); From 0b85867a644e3d1ff6990f1506466c0bcb827e5c Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 8 Jun 2020 18:04:50 +0800 Subject: [PATCH 222/378] =?UTF-8?q?redis=20=20sink=20=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/redis/RedisOutputFormat.java | 79 +++++++------------ .../flink/sql/sink/redis/enums/RedisType.java | 33 ++++++++ 2 files changed, 62 insertions(+), 50 deletions(-) create mode 100644 redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ae4fe5a4b..ab97cf60e 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -19,6 +19,8 @@ package com.dtstack.flink.sql.sink.redis; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.sink.redis.enums.RedisType; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -35,11 +37,7 @@ import java.io.Closeable; import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; /** * @author yanxi @@ -49,7 +47,7 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { private String url; - private String database; + private String database = "0"; private String tableName; @@ -71,7 +69,7 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { protected List primaryKeys; - protected int timeout; + protected int timeout = 10000; private JedisPool pool; @@ -121,29 +119,21 @@ private void establishConnection() { String[] ipPortPair = StringUtils.split(ipPort, ":"); addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); } - if (timeout == 0){ - timeout = 10000; - } - if (database == null) - { - database = "0"; - } - switch (redisType){ - //单机 - case 1: + switch (RedisType.parse(redisType)){ + case STANDALONE: pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); jedis = pool.getResource(); break; - //哨兵 - case 2: + case SENTINEL: jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; - //集群 - case 3: + case CLUSTER: jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + break; default: + throw new RuntimeException("unsupport redis type[ " + redisType + "]"); } } @@ -158,36 +148,14 @@ public void writeRecord(Tuple2 record) throws IOException { if (row.getArity() != fieldNames.length) { return; } - - HashMap map = new HashMap<>(8); - for (String primaryKey : primaryKeys) { - for (int i = 0; i < fieldNames.length; i++) { - if (fieldNames[i].equals(primaryKey)) { - map.put(primaryKey, i); - } - } - } - - List kvList = new LinkedList<>(); - for (String primaryKey : primaryKeys){ - StringBuilder primaryKv = new StringBuilder(); - int index = map.get(primaryKey).intValue(); - primaryKv.append(primaryKey).append(":").append(row.getField(index)); - kvList.add(primaryKv.toString()); - } - - String perKey = String.join(":", kvList); - for (int i = 0; i < fieldNames.length; i++) { - StringBuilder key = new StringBuilder(); - key.append(tableName).append(":").append(perKey).append(":").append(fieldNames[i]); - - String value = "null"; - Object field = row.getField(i); - if (field != null) { - value = field.toString(); - } - jedis.set(key.toString(), value); + Map refData = Maps.newHashMap(); + for(int i = 0; i < fieldNames.length; i++){ + refData.put(fieldNames[i], row.getField(i)); } + String redisKey = buildCacheKey(refData); + refData.entrySet().forEach(e ->{ + jedis.hset(redisKey, e.getKey(), String.valueOf(e.getValue())); + }); if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ LOG.info(record.toString()); @@ -211,6 +179,17 @@ public void close() throws IOException { } + public String buildCacheKey(Map refData) { + StringBuilder keyBuilder = new StringBuilder(tableName); + for(String primaryKey : primaryKeys){ + if(!refData.containsKey(primaryKey)){ + return null; + } + keyBuilder.append("_").append(refData.get(primaryKey)); + } + return keyBuilder.toString(); + } + public static RedisOutputFormatBuilder buildRedisOutputFormat(){ return new RedisOutputFormatBuilder(); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java new file mode 100644 index 000000000..7a4054dfc --- /dev/null +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/enums/RedisType.java @@ -0,0 +1,33 @@ +package com.dtstack.flink.sql.sink.redis.enums; + +public enum RedisType { + /** + * 单机 + */ + STANDALONE(1), + /** + * 哨兵 + */ + SENTINEL(2), + /** + * 集群 + */ + CLUSTER(3); + int type; + RedisType(int type){ + this.type = type; + } + + public int getType(){ + return type; + } + + public static RedisType parse(int redisType){ + for(RedisType type : RedisType.values()){ + if(type.getType() == redisType){ + return type; + } + } + throw new RuntimeException("unsupport redis type["+ redisType + "]"); + } +} From 7fafc922a26c5eb5aff114bfd44c6041c07307bc Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 8 Jun 2020 19:27:55 +0800 Subject: [PATCH 223/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E6=83=85=E5=86=B5=E4=B8=8B=E3=80=82=E5=A6=82?= =?UTF-8?q?=E6=9E=9C=E5=AD=97=E6=AE=B5=E9=80=89=E6=8B=A9=E4=BA=86ROWTIME,?= =?UTF-8?q?=20=E6=A1=86=E6=9E=B6=E6=9C=AC=E8=BA=AB=E5=9C=A8=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E7=9A=84=E6=97=B6=E5=80=99=E4=BC=9A=E5=AF=B9rowtime?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E6=97=B6=E5=8C=BA=E5=A4=84=E7=90=86=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E8=87=B4window=E6=97=A0=E6=B3=95=E8=A7=A6=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/cassandra/CassandraAsyncReqRow.java | 6 +++++- .../side/elasticsearch6/Elasticsearch6AsyncReqRow.java | 5 ++++- .../dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java | 6 +++++- .../dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java | 10 ++++++---- .../dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java | 10 ++++++---- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 6 +++++- .../flink/sql/side/redis/table/RedisSideReqRow.java | 6 +++++- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index ef4380c7a..5197284e1 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -60,6 +60,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.TimeZone; /** * Reason: @@ -73,6 +74,8 @@ public class CassandraAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncReqRow.class); + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 10; private final static int DEFAULT_VERTX_WORKER_POOL_SIZE = 20; @@ -251,7 +254,8 @@ public Row fillData(Row input, Object line) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index 502f23f0a..fa6d507bb 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -57,6 +57,7 @@ import java.sql.Timestamp; import java.util.List; import java.util.Map; +import java.util.TimeZone; /** * @author yinxi @@ -65,6 +66,7 @@ public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch6AsyncReqRow.class); + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); @@ -195,7 +197,8 @@ public Row fillData(Row input, Object line) { Object obj = input.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 56f50e27c..ed7f53e4d 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -48,6 +48,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -63,6 +64,8 @@ public class HbaseAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635104857937717L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private static final Logger LOG = LoggerFactory.getLogger(HbaseAsyncReqRow.class); //match to the rule of netty3 @@ -143,7 +146,8 @@ public Row fillData(Row input, Object sideInput){ boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if(obj instanceof Timestamp && isTimeIndicatorTypeInfo){ - obj = ((Timestamp)obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index e3200c293..edb49814f 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -36,14 +36,14 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; public class KuduAsyncReqRow extends BaseAsyncReqRow { private static final Logger LOG = LoggerFactory.getLogger(KuduAsyncReqRow.class); + + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + /** * 获取连接的尝试次数 */ @@ -184,6 +184,8 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); } diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 0cf8158c7..f19179602 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -50,10 +50,7 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; /** @@ -63,8 +60,11 @@ * @author xuqianjin */ public class MongoAsyncReqRow extends BaseAsyncReqRow { + private static final long serialVersionUID = -1183158242862673706L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private static final Logger LOG = LoggerFactory.getLogger(MongoAsyncReqRow.class); private transient MongoClient mongoClient; @@ -165,6 +165,8 @@ public Row fillData(Row input, Object line) { if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..3fb63410a 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -47,6 +47,7 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -62,6 +63,8 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private static final long serialVersionUID = 2098635244857937720L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private static final Logger LOG = LoggerFactory.getLogger(RdbAsyncReqRow.class); public final static int DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE = 1; @@ -211,7 +214,8 @@ public Row fillData(Row input, Object line) { Object obj = input.getField(entry.getValue()); boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 17557f916..634cc66fa 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -28,6 +28,7 @@ import java.sql.Date; import java.sql.Timestamp; import java.util.Map; +import java.util.TimeZone; /** * redis fill row data @@ -40,6 +41,8 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + private BaseSideInfo sideInfo; public RedisSideReqRow(BaseSideInfo sideInfo){ @@ -55,7 +58,8 @@ public Row fillData(Row input, Object sideInput) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if(obj instanceof Timestamp && isTimeIndicatorTypeInfo){ - obj = ((Timestamp)obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); } From eee1867ec18dd68f8527b2180cb0592b548dae12 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 8 Jun 2020 21:27:39 +0800 Subject: [PATCH 224/378] =?UTF-8?q?=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plugin/redisSink.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/plugin/redisSink.md b/docs/plugin/redisSink.md index 103cb4997..eb0109f06 100644 --- a/docs/plugin/redisSink.md +++ b/docs/plugin/redisSink.md @@ -94,7 +94,8 @@ redis5.0 ## 6.redis完整样例 ### redis数据说明 -redis使用k-v格式存储，key的构建格式为tableName:privateKey:privateKeyValue:columnName, value=columnValue +redis使用散列类型 hash 数据结构，key=tableName_primaryKey1_primaryKey2,value={column1=value1, column2=value2} +如果以班级class表为例，id和name作为联合主键，那么redis的结构为 ### 源表数据内容 ``` @@ -103,10 +104,10 @@ redis使用k-v格式存储，key的构建格式为tableName:privateKey:privateKe ### redis实际数据内容 ``` 127.0.0.1:6379> keys * -1) "resultTable:name:roc:name" -2) "resultTable:name:roc:channel" -127.0.0.1:6379> get "resultTable:name:roc:name" -"roc" -127.0.0.1:6379> get "resultTable:name:roc:channel" -"daishu" +1) "resultTable_roc" +127.0.0.1:6379> hgetall resultTable_roc +1) "channel" +2) "daishu" +3) "name" +4) "roc" ``` \ No newline at end of file From df063184819ee48b10068b38ddaee8b1426af066 Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 8 Jun 2020 21:37:09 +0800 Subject: [PATCH 225/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BB=B4=E8=A1=A8All?= =?UTF-8?q?Cache=20=E6=83=85=E5=86=B5=E4=B8=8B=E3=80=82=E5=A6=82=E6=9E=9C?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E9=80=89=E6=8B=A9=E4=BA=86ROWTIME,=20?= =?UTF-8?q?=E6=A1=86=E6=9E=B6=E6=9C=AC=E8=BA=AB=E5=9C=A8=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=E4=BC=9A=E5=AF=B9rowtime=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=97=B6=E5=8C=BA=E5=A4=84=E7=90=86=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4window=E6=97=A0=E6=B3=95=E8=A7=A6=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/cassandra/CassandraAllReqRow.java | 9 ++++----- .../java/com/dtstack/flink/sql/side/BaseAllReqRow.java | 3 +++ .../sql/side/elasticsearch6/Elasticsearch6AllReqRow.java | 4 +++- .../com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java | 4 +++- .../com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java | 4 +++- .../com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java | 4 +++- .../flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 8 +++----- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index 5f642d176..2fd33b4f9 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -50,10 +50,7 @@ import java.net.InetAddress; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; /** @@ -91,8 +88,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java index ad9dcf1bd..28f678f10 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAllReqRow.java @@ -32,6 +32,7 @@ import org.apache.calcite.sql.JoinType; import java.sql.SQLException; +import java.util.TimeZone; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -49,6 +50,8 @@ public abstract class BaseAllReqRow extends RichFlatMapFunction impl public static final long LOAD_DATA_ERROR_SLEEP_TIME = 5_000L; + public static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + protected BaseSideInfo sideInfo; private ScheduledExecutorService es; diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java index e50ed5edc..bc60eb30a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-all-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AllReqRow.java @@ -106,9 +106,11 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 63d26d5dd..63dc5cb29 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -88,8 +88,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if(obj instanceof Timestamp && isTimeIndicatorTypeInfo){ - obj = ((Timestamp)obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 31f9aa13d..cf7f0dea7 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -76,8 +76,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 1857dbede..b3e1b3e9d 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -88,8 +88,10 @@ public Row fillData(Row input, Object sideInput) { //Type information for indicating event or processing time. However, it behaves like a regular SQL timestamp but is serialized as Long. if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } + row.setField(entry.getKey(), obj); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index bb88c7787..afdcac63a 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -43,10 +43,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -161,7 +158,8 @@ public Row fillData(Row input, Object sideInput) { protected Object dealTimeAttributeType(Class entry, Object obj) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { - obj = ((Timestamp) obj).getTime(); + //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 + obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } return obj; } From 3339014f1dc2f16da2fc4f28a9f79f3dce6fe58b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 10 Jun 2020 11:15:51 +0800 Subject: [PATCH 226/378] add tidb --- tidb/pom.xml | 47 ++++++++++ tidb/tidb-side/pom.xml | 36 +++++++ tidb/tidb-side/tidb-all-side/pom.xml | 91 ++++++++++++++++++ .../flink/sql/side/tidb/TidbAllReqRow.java | 71 ++++++++++++++ .../flink/sql/side/tidb/TidbAllSideInfo.java | 43 +++++++++ tidb/tidb-side/tidb-async-side/pom.xml | 92 ++++++++++++++++++ .../flink/sql/side/tidb/TidbAsyncReqRow.java | 72 ++++++++++++++ .../sql/side/tidb/TidbAsyncSideInfo.java | 44 +++++++++ tidb/tidb-side/tidb-side-core/pom.xml | 16 ++++ .../sql/side/tidb/table/TidbSideParser.java | 41 ++++++++ tidb/tidb-sink/pom.xml | 93 +++++++++++++++++++ .../flink/sql/sink/tidb/TidbDialect.java | 75 +++++++++++++++ .../dtstack/flink/sql/sink/tidb/TidbSink.java | 58 ++++++++++++ .../sql/sink/tidb/table/TidbSinkParser.java | 41 ++++++++ 14 files changed, 820 insertions(+) create mode 100644 tidb/pom.xml create mode 100644 tidb/tidb-side/pom.xml create mode 100644 tidb/tidb-side/tidb-all-side/pom.xml create mode 100644 tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java create mode 100644 tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java create mode 100644 tidb/tidb-side/tidb-async-side/pom.xml create mode 100644 tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java create mode 100644 tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java create mode 100644 tidb/tidb-side/tidb-side-core/pom.xml create mode 100644 tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java create mode 100644 tidb/tidb-sink/pom.xml create mode 100644 tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java create mode 100644 tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java create mode 100644 tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java diff --git a/tidb/pom.xml b/tidb/pom.xml new file mode 100644 index 000000000..851acba6b --- /dev/null +++ b/tidb/pom.xml @@ -0,0 +1,47 @@ + + + + flink.sql + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.tidb + pom + + tidb-side + tidb-sink + + + + 5.1.46 + 1.0-SNAPSHOT + + + + + junit + junit + 3.8.1 + test + + + + com.dtstack.flink + sql.core + ${sql.core.version} + provided + + + + mysql + mysql-connector-java + ${tidb.connector.version} + + + + diff --git a/tidb/tidb-side/pom.xml b/tidb/tidb-side/pom.xml new file mode 100644 index 000000000..18c2b8402 --- /dev/null +++ b/tidb/tidb-side/pom.xml @@ -0,0 +1,36 @@ + + + + sql.tidb + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.tidb + 1.0-SNAPSHOT + + tidb-all-side + tidb-side-core + tidb-async-side + + tidb-side + pom + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.rdb + ${rdb.side.version} + + + + + diff --git a/tidb/tidb-side/tidb-all-side/pom.xml b/tidb/tidb-side/tidb-all-side/pom.xml new file mode 100644 index 000000000..e158fe9f5 --- /dev/null +++ b/tidb/tidb-side/tidb-all-side/pom.xml @@ -0,0 +1,91 @@ + + + + sql.side.tidb + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.all.tidb + tidb-all-side + jar + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.tidb.core + ${sql.side.tidb.core.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java new file mode 100644 index 000000000..c1529cd63 --- /dev/null +++ b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllReqRow.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; +import com.dtstack.flink.sql.util.DtStringUtil; +import com.google.common.collect.Maps; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAllReqRow extends AbstractRdbAllReqRow { + + private static final Logger LOG = LoggerFactory.getLogger(TidbAllReqRow.class); + + private static final String TIDB_DRIVER = "com.mysql.jdbc.Driver"; + + public TidbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(new TidbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public Connection getConn(String dbUrl, String userName, String password) { + try { + Class.forName(TIDB_DRIVER); + Map addParams = Maps.newHashMap(); + addParams.put("userCursorFetch", "true"); + String targetDbUrl = DtStringUtil.addJdbcParam(dbUrl, addParams, true); + return DriverManager.getConnection(targetDbUrl, userName, password); + } catch (ClassNotFoundException | SQLException e) { + LOG.error("TiDB get connect error!", e); + throw new RuntimeException("TiDB get connect error! ", e); + + } + } + + @Override + public int getFetchSize() { + return Integer.MIN_VALUE; + } +} diff --git a/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java new file mode 100644 index 000000000..2a7a39076 --- /dev/null +++ b/tidb/tidb-side/tidb-all-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAllSideInfo.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAllSideInfo extends RdbAllSideInfo { + public TidbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } +} diff --git a/tidb/tidb-side/tidb-async-side/pom.xml b/tidb/tidb-side/tidb-async-side/pom.xml new file mode 100644 index 000000000..bdb5ec521 --- /dev/null +++ b/tidb/tidb-side/tidb-async-side/pom.xml @@ -0,0 +1,92 @@ + + + + sql.side.tidb + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.side.async.tidb + 1.0-SNAPSHOT + tidb-async-side + jar + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.side.tidb.core + ${sql.side.tidb.core.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java new file mode 100644 index 000000000..c4c712be5 --- /dev/null +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.jdbc.JDBCClient; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; + +import java.util.List; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAsyncReqRow extends RdbAsyncReqRow { + + private final static String TIDB_DRIVER = "com.mysql.jdbc.Driver"; + + public TidbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(new TidbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + JsonObject tidbClientConfig = new JsonObject(); + RdbSideTableInfo sideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); + tidbClientConfig.put("url", sideTableInfo.getUrl()) + .put("driver_class", TIDB_DRIVER) + .put("max_pool_size", sideTableInfo.getAsyncPoolSize()) + .put("user", sideTableInfo.getUserName()) + .put("password", sideTableInfo.getPassword()) + .put("provider_class", DT_PROVIDER_CLASS) + .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN) + .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID); + + System.setProperty("vertx.disableFileCPResolving", "true"); + + VertxOptions vo = new VertxOptions(); + vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); + vo.setWorkerPoolSize(sideTableInfo.getAsyncPoolSize()); + vo.setFileResolverCachingEnabled(false); + Vertx vertx = Vertx.vertx(vo); + setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); + } +} diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java new file mode 100644 index 000000000..f672f0b83 --- /dev/null +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncSideInfo.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb; + +import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import org.apache.flink.api.java.typeutils.RowTypeInfo; + +import java.util.List; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbAsyncSideInfo extends RdbAsyncSideInfo { + + public TidbAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } +} diff --git a/tidb/tidb-side/tidb-side-core/pom.xml b/tidb/tidb-side/tidb-side-core/pom.xml new file mode 100644 index 000000000..201dc9198 --- /dev/null +++ b/tidb/tidb-side/tidb-side-core/pom.xml @@ -0,0 +1,16 @@ + + + + sql.side.tidb + com.dtstack.flink + 1.0-SNAPSHOT + + 4.0.0 + + sql.side.tidb.core + jar + tidb-side-core + 1.0-SNAPSHOT + \ No newline at end of file diff --git a/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java b/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java new file mode 100644 index 000000000..ae8f59686 --- /dev/null +++ b/tidb/tidb-side/tidb-side-core/src/main/java/com/dtstack/flink/sql/side/tidb/table/TidbSideParser.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.side.tidb.table; + +import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbSideParser extends RdbSideParser { + + private static final String CURRENT_TYPE = "tidb"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + tableInfo.setType(CURRENT_TYPE); + return tableInfo; + } +} diff --git a/tidb/tidb-sink/pom.xml b/tidb/tidb-sink/pom.xml new file mode 100644 index 000000000..2eaad0e99 --- /dev/null +++ b/tidb/tidb-sink/pom.xml @@ -0,0 +1,93 @@ + + + + sql.tidb + com.dtstack.flink + 1.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + sql.sink.tidb + tidb-sink + jar + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + + + + + com.dtstack.flink + sql.sink.rdb + ${sql.sink.rdb.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + false + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + 1.2 + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java new file mode 100644 index 000000000..3a7770ede --- /dev/null +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbDialect.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.tidb; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbDialect implements JDBCDialect { + private static final long serialVersionUID = 1L; + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:mysql:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("com.mysql.jdbc.Driver"); + } + + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + /** + * 根据ALLReplace参数，选择使用replace语句还是ON DUPLICATE KEY UPDATE 语句 + */ + @Override + public Optional getUpsertStatement(String schema, String tableName, String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { + return allReplace ? buildReplaceIntoStatement(tableName, fieldNames) : buildDuplicateUpsertStatement(tableName, fieldNames); + } + + public Optional buildDuplicateUpsertStatement(String tableName, String[] fieldNames) { + String updateClause = Arrays.stream(fieldNames).map(f -> quoteIdentifier(f) + "=IFNULL(VALUES(" + quoteIdentifier(f) + ")," + quoteIdentifier(f) + ")") + .collect(Collectors.joining(", ")); + return Optional.of(getInsertIntoStatement("", tableName, fieldNames, null) + + " ON DUPLICATE KEY UPDATE " + updateClause); + } + + public Optional buildReplaceIntoStatement(String tableName, String[] fieldNames) { + String columns = Arrays.stream(fieldNames) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fieldNames) + .map(f -> "?") + .collect(Collectors.joining(", ")); + return Optional.of("REPLACE INTO " + quoteIdentifier(tableName) + + "(" + columns + ")" + " VALUES (" + placeholders + ")"); + } +} diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java new file mode 100644 index 000000000..a8b031a17 --- /dev/null +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/TidbSink.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.tidb; + +import com.dtstack.flink.sql.sink.IStreamSinkGener; +import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; +import com.dtstack.flink.sql.sink.rdb.JDBCOptions; +import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbSink extends AbstractRdbSink implements IStreamSinkGener { + + public TidbSink() { + super(new TidbDialect()); + } + + @Override + public JDBCUpsertOutputFormat getOutputFormat() { + JDBCOptions jdbcOptions = JDBCOptions.builder() + .setDbUrl(dbUrl) + .setDialect(jdbcDialect) + .setUsername(userName) + .setPassword(password) + .setTableName(tableName) + .build(); + + return JDBCUpsertOutputFormat.builder() + .setOptions(jdbcOptions) + .setFieldNames(fieldNames) + .setFlushMaxSize(batchNum) + .setFlushIntervalMills(batchWaitInterval) + .setFieldTypes(sqlTypes) + .setKeyFields(primaryKeys) + .setAllReplace(allReplace) + .setUpdateMode(updateMode) + .build(); + } +} diff --git a/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java new file mode 100644 index 000000000..d00206cc2 --- /dev/null +++ b/tidb/tidb-sink/src/main/java/com/dtstack/flink/sql/sink/tidb/table/TidbSinkParser.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.tidb.table; + +import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; +import com.dtstack.flink.sql.table.AbstractTableInfo; + +import java.util.Map; + +/** + * @author tiezhu + * Date 2020/6/1 + * company www.dtstack.com + */ +public class TidbSinkParser extends RdbSinkParser { + + private static final String CURRENT_TYPE = "tidb"; + + @Override + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + AbstractTableInfo tableInfo = super.getTableInfo(tableName, fieldsInfo, props); + tableInfo.setType(CURRENT_TYPE); + return tableInfo; + } +} From 64afbc6651202fbdb4d93f7f548cff591c07b242 Mon Sep 17 00:00:00 2001 From: gkd Date: Wed, 10 Jun 2020 16:30:43 +0800 Subject: [PATCH 227/378] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B7=BB=E5=8A=A0che?= =?UTF-8?q?ck()=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java | 2 +- .../com/dtstack/flink/sql/sink/mongo/table/MongoSinkParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java index 4fe1ebee1..99bec0b74 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideParser.java @@ -59,7 +59,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 10 Jun 2020 20:06:26 +0800 Subject: [PATCH 228/378] =?UTF-8?q?=E7=94=A8isValid=E4=BB=A3=E6=9B=BFisClo?= =?UTF-8?q?sed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/rdb/format/JDBCUpsertOutputFormat.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index e8b6dc8a4..4e047492f 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -33,7 +33,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -44,6 +46,7 @@ /** * An upsert OutputFormat for JDBC. + * * @author maqi */ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat> { @@ -103,7 +106,7 @@ public JDBCUpsertOutputFormat( * * @param taskNumber The number of the parallel instance. * @throws IOException Thrown, if the output could not be opened due to an - * I/O problem. + * I/O problem. */ @Override public void open(int taskNumber, int numTasks) throws IOException { @@ -167,7 +170,7 @@ public synchronized void writeRecord(Tuple2 tuple2) throws IOExcep private void checkConnectionOpen() { try { - if (connection.isClosed()) { + if (!connection.isValid(10)) { LOG.info("db connection reconnect.."); establishConnection(); jdbcWriter.prepareStatement(connection); @@ -270,7 +273,8 @@ public Builder setFieldTypes(int[] fieldTypes) { } /** - * optional, partition Fields + * optional, partition Fields + * * @param partitionFields * @return */ From 4e747f5537fef5ac53e79b1d12c11302504ab299 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 11 Jun 2020 17:06:47 +0800 Subject: [PATCH 229/378] fix --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 87c8e97e6..374ec3c31 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ db2 polardb oceanbase + tidb From c0a56ef2c41f30708f1c9565977b50aabe54f33e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 11 Jun 2020 18:02:12 +0800 Subject: [PATCH 230/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index c4c712be5..643b6368d 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.tidb; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -31,6 +32,9 @@ import org.apache.flink.configuration.Configuration; import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * @author tiezhu @@ -68,5 +72,7 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); + setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 512320f7604267e908999f6b2ade01e1f4685f1e Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 12 Jun 2020 15:09:52 +0800 Subject: [PATCH 231/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9fastjson=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E4=B8=BA1.2.70?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launcher/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/pom.xml b/launcher/pom.xml index ea921d87a..df9e8ae29 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -38,7 +38,7 @@ com.alibaba fastjson - 1.2.7 + 1.2.70 From 20635f032ac8d1e77df893079682df0594c6dd55 Mon Sep 17 00:00:00 2001 From: gkd Date: Mon, 15 Jun 2020 12:51:18 +0800 Subject: [PATCH 232/378] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20topic?= =?UTF-8?q?=E4=B8=8D=E8=AE=BE=E7=BD=AE=E6=8A=A5NPE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 12 ++++++++++++ .../sql/source/kafka/table/KafkaSourceParser.java | 2 -- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index be1020b04..a5381da8d 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,6 +122,18 @@ junit 4.12 + + log4j + log4j + 1.2.17 + + + + org.slf4j + slf4j-simple + 1.7.30 + + diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index bef86f10d..00424127d 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -48,10 +48,8 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 15 Jun 2020 13:05:16 +0800 Subject: [PATCH 233/378] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20topic?= =?UTF-8?q?=E4=B8=8D=E8=AE=BE=E7=BD=AE=E6=8A=A5NPE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 00424127d..10cdf27e7 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -51,7 +51,6 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Mon, 15 Jun 2020 13:08:22 +0800 Subject: [PATCH 234/378] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka=20topic?= =?UTF-8?q?=E4=B8=8D=E8=AE=BE=E7=BD=AE=E6=8A=A5NPE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 10cdf27e7..4a84223cc 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -50,6 +50,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Tue, 16 Jun 2020 10:28:31 +0800 Subject: [PATCH 235/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9hadoop(?= =?UTF-8?q?=E9=9D=9EKerberos)=20=E5=86=99=E5=85=A5=20hbase=EF=BC=88?= =?UTF-8?q?=E5=BC=80=E5=90=AFkerberos=EF=BC=89=E5=85=BC=E5=AE=B9=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 12 ++++++++++++ .../flink/sql/sink/hbase/HbaseConfigUtils.java | 18 ++++++++++++++++-- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index be1020b04..2db344e64 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -108,6 +108,18 @@ org.apache.flink flink-yarn_2.11 ${flink.version} + + + org.apache.flink + flink-shaded-hadoop2 + + + + + + org.apache.flink + flink-shaded-hadoop2 + 2.7.5-1.8.1 diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index 57c63d243..7117644db 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -19,9 +19,13 @@ package com.dtstack.flink.sql.sink.hbase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.HadoopKerberosName; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.security.krb5.Config; +import sun.security.krb5.KrbException; import java.io.IOException; @@ -54,7 +58,11 @@ public class HbaseConfigUtils { public static final String KEY_JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; public static final String KEY_ZOOKEEPER_SASL_CLIENT = "zookeeper.sasl.client"; - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { + public static final String KEY_HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + public static final String KEY_HADOOP_SECURITY_AUTH_TO_LOCAL = "hadoop.security.auth_to_local"; + public static final String KEY_HADOOP_SECURITY_AUTHORIZATION = "hadoop.security.authorization"; + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException, KrbException { LOG.info("loginAndReturnUGI principal {}",principal); LOG.info("loginAndReturnUGI keytab {}",keytab); if (conf == null) { @@ -69,7 +77,13 @@ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String throw new IllegalArgumentException("keytab can not be null"); } - conf.set("hadoop.security.authentication", "Kerberos"); + conf.set(KEY_HADOOP_SECURITY_AUTHENTICATION, "Kerberos"); + //conf.set("hadoop.security.auth_to_local", "DEFAULT"); + conf.set(KEY_HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[1:$1] RULE:[2:$1]"); + conf.set(KEY_HADOOP_SECURITY_AUTHORIZATION, "true"); + + Config.refresh(); + KerberosName.resetDefaultRealm(); UserGroupInformation.setConfiguration(conf); return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a6a889cc4..222d2b052 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -112,7 +112,7 @@ private void openConn(){ } } - private void openKerberosConn() throws IOException { + private void openKerberosConn() throws Exception { conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, host); conf.set(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, zkParent); From b50c63c00b49b5b715516e2050ad98cbb21ecc2b Mon Sep 17 00:00:00 2001 From: gkd Date: Tue, 16 Jun 2020 17:25:05 +0800 Subject: [PATCH 236/378] =?UTF-8?q?=E5=8E=BB=E6=8E=89log4j?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/pom.xml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index a5381da8d..e98800402 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,17 +122,6 @@ junit 4.12 - - log4j - log4j - 1.2.17 - - - - org.slf4j - slf4j-simple - 1.7.30 - From 3bb095cdfc9cd7f98474e35af322d049b3b699e1 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 17 Jun 2020 19:22:13 +0800 Subject: [PATCH 237/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20--=20=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20=E5=B9=B6=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/parser/SqlParser.java | 2 +- .../dtstack/flink/sql/util/DtStringUtil.java | 51 +++++++++++++++++++ .../flink/sql/util/TestDtStringUtil.java | 38 ++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 2afc76c48..23f8e4942 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -68,7 +68,7 @@ public static SqlTree parseSql(String sql) throws Exception { throw new RuntimeException("need to set local sql plugin root"); } - sql = sql.replaceAll("--.*", "") + sql = DtStringUtil.dealSqlComment(sql) .replaceAll("\r\n", " ") .replaceAll("\n", " ") .replace("\t", " ").trim(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 3106a1bd5..b4200d733 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -106,6 +106,57 @@ public static String replaceIgnoreQuota(String str, String oriStr, String replac return str.replaceAll(splitPatternStr, replaceStr); } + /** + * 处理 sql 中 "--" 注释，而不删除引号内的内容 + * + * @param sql 解析出来的 sql + * @return 返回无注释内容的 sql + */ + public static String dealSqlComment(String sql) { + boolean inQuotes = false; + boolean inSingleQuotes = false; + int bracketLeftNum = 0; + StringBuilder b = new StringBuilder(sql.length()); + char[] chars = sql.toCharArray(); + for (int index = 0; index < chars.length; index ++) { + if (index == chars.length) { + return b.toString(); + } + StringBuilder tempSb = new StringBuilder(2); + if (index > 1) { + tempSb.append(chars[index - 1]); + tempSb.append(chars[index]); + } + + if (tempSb.toString().equals("--")) { + if (inQuotes) { + b.append(chars[index]); + } else if (inSingleQuotes) { + b.append(chars[index]); + } else if (bracketLeftNum > 0) { + b.append(chars[index]); + } else { + b.deleteCharAt(b.length() - 1); + while (chars[index] != '\n') { + // 判断注释内容是不是行尾或者 sql 的最后一行 + if (index == chars.length - 1) { + break; + } + index++; + } + } + } else if (chars[index] == '\"' && '\\' != chars[index] && !inSingleQuotes) { + inQuotes = !inQuotes; + b.append(chars[index]); + } else if (chars[index] == '\'' && '\\' != chars[index] && !inQuotes) { + inSingleQuotes = !inSingleQuotes; + b.append(chars[index]); + } else { + b.append(chars[index]); + } + } + return b.toString(); + } public static String col2string(Object column, String type) { String rowData = column.toString(); diff --git a/core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java b/core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java new file mode 100644 index 000000000..cf3fdb90b --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/util/TestDtStringUtil.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.junit.Assert; +import org.junit.Test; + +/** + * @author tiezhu + * Date 2020/6/17 星期三 + */ +public class TestDtStringUtil { + @Test + public void dealSqlCommentTest() { + String testSQLWithComment = "CREATE TABLE MyTable --this is a comment\n"; + Assert.assertEquals("CREATE TABLE MyTable ", DtStringUtil.dealSqlComment(testSQLWithComment)); + testSQLWithComment = "CREATE TABLE 'MyTable--' --this is a comment"; + Assert.assertEquals("CREATE TABLE 'MyTable--' ", DtStringUtil.dealSqlComment(testSQLWithComment)); + testSQLWithComment = "CREATE TABLE MyTable -- this is a '--comment'\n"; + Assert.assertEquals("CREATE TABLE MyTable ", DtStringUtil.dealSqlComment(testSQLWithComment)); + } +} From 86b625526a69023b1b938d8e9fbb85f6d5a8ed24 Mon Sep 17 00:00:00 2001 From: xuchao Date: Fri, 19 Jun 2020 13:59:55 +0800 Subject: [PATCH 238/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9sonar=20=E7=9A=84?= =?UTF-8?q?=E9=92=89=E9=92=89=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ci/sonar_notify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sonar_notify.sh b/ci/sonar_notify.sh index 41f8a3c0e..da36b8c5e 100644 --- a/ci/sonar_notify.sh +++ b/ci/sonar_notify.sh @@ -1,7 +1,7 @@ #!/bin/bash #参考钉钉文档 https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq sonarreport=$(curl -s http://172.16.100.198:8082/?projectname=dt-insight-engine/flinkStreamSQL) - curl -s "https://oapi.dingtalk.com/robot/send?access_token=71555061297a53d3ac922a6f4d94285d8e23bccdca0c00b4dc6df0a2d49da724" \ + curl -s "https://oapi.dingtalk.com/robot/send?access_token=58fd731d8bed3b17708d3aa27e49a7e2c41c7e6545f6c4be3170963a7bba7e2a" \ -H "Content-Type: application/json" \ -d "{ \"msgtype\": \"markdown\", From c931b0b39871ff391b0249170dbfd53dadfc94b3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 19 Jun 2020 14:34:27 +0800 Subject: [PATCH 239/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9sql=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=9A=84=E9=BB=98=E8=AE=A4=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index b60a794b1..ab3e0517b 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -56,7 +56,7 @@ public void open(Configuration parameters) throws Exception { .put("user", rdbSideTableInfo.getUserName()) .put("password", rdbSideTableInfo.getPassword()) .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) + .put("preferred_test_query", "select 1 from dual") .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..434d1b3c0 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -78,7 +78,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static String DT_PROVIDER_CLASS = "com.dtstack.flink.sql.side.rdb.provider.DTC3P0DataSourceProvider"; - public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; + public final static String PREFERRED_TEST_QUERY_SQL = "select 1"; private transient SQLClient rdbSqlClient; From a831177893c16154f7d04db625cc4a708be6f907 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 19 Jun 2020 16:47:57 +0800 Subject: [PATCH 240/378] =?UTF-8?q?=E7=BB=B4=E8=A1=A8all=E7=9A=84=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 8 ++++---- .../flink/sql/side/hbase/HbaseAllSideInfo.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 63d26d5dd..5f60ce690 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,6 +25,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; +import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -165,6 +166,7 @@ public void flatMap(CRow input, Collector out) throws Exception { private void loadData(Map> tmpCache) throws SQLException { AbstractSideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); + Map colRefType = ((HbaseAllSideInfo)sideInfo).getColRefType(); HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum", hbaseSideTableInfo.getHost()); @@ -177,14 +179,12 @@ private void loadData(Map> tmpCache) throws SQLExcep resultScanner = table.getScanner(new Scan()); for (Result r : resultScanner) { Map kv = new HashedMap(); - for (Cell cell : r.listCells()) - { + for (Cell cell : r.listCells()) { String family = Bytes.toString(CellUtil.cloneFamily(cell)); String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell)); - String value = Bytes.toString(CellUtil.cloneValue(cell)); StringBuilder key = new StringBuilder(); key.append(family).append(":").append(qualifier); - + Object value = HbaseUtils.convertByte(CellUtil.cloneValue(cell), colRefType.get(key.toString())); kv.put(aliasNameInversion.get(key.toString()), value); } tmpCache.put(new String(r.getRow()), kv); diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index 663c2927b..1b1da02ab 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -24,17 +24,22 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.BaseSideInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; +import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.util.ParseUtils; +import com.google.common.collect.Maps; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; import com.google.common.collect.Lists; import java.util.List; +import java.util.Map; public class HbaseAllSideInfo extends BaseSideInfo { private RowKeyBuilder rowKeyBuilder; + private Map colRefType; + public HbaseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -48,6 +53,14 @@ public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInf rowKeyBuilder.init(sideTableInfo.getPrimaryKeys().get(0)); + HbaseSideTableInfo hbaseSideTableInfo = (HbaseSideTableInfo) sideTableInfo; + colRefType = Maps.newHashMap(); + for(int i=0; i getColRefType() { + return colRefType; + } + } \ No newline at end of file From 5693ad448a6ae5b57136b3278f605a1fdd38124a Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 19 Jun 2020 17:07:32 +0800 Subject: [PATCH 241/378] =?UTF-8?q?=E7=BB=93=E6=9E=9C=E8=A1=A8=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/hbase/HbaseOutputFormat.java | 27 +------------------ .../flink/sql/sink/hbase/HbaseUtil.java | 5 ++++ 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index a3189b0a6..e88324936 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -92,13 +92,8 @@ public void open(int taskNumber, int numTasks) throws IOException { @Override public void writeRecord(Tuple2 tuple2) { Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.f0; Row row = tupleTrans.f1; - if (retract) { - dealInsert(row); - } else if (!retract && StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { - dealDelete(row); - } + dealInsert(row); } protected void dealInsert(Row record) { @@ -123,26 +118,6 @@ protected void dealInsert(Row record) { outRecords.inc(); } - protected void dealDelete(Row record) { - String rowKey = buildRowKey(record); - if (!StringUtils.isEmpty(rowKey)) { - Delete delete = new Delete(Bytes.toBytes(rowKey)); - try { - table.delete(delete); - } catch (IOException e) { - if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..{}", record.toString()); - LOG.error("", e); - } - outDirtyRecords.inc(); - } - if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { - LOG.info(record.toString()); - } - outRecords.inc(); - } - } - private Put getPutByRow(Row record) { String rowKey = buildRowKey(record); if (StringUtils.isEmpty(rowKey)) { diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java index 8f1166574..6191bd177 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java @@ -64,4 +64,9 @@ public static TypeInformation columnTypeToTypeInformation(String type) { } + public static byte[] toByte(Object value){ + if() + return new byte[]; + } + } From 83e919a1697c34103aef9409ddda7b56ffb87f89 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 19 Jun 2020 21:37:46 +0800 Subject: [PATCH 242/378] =?UTF-8?q?=E8=A7=A3=E5=86=B3join=E7=9B=B8?= =?UTF-8?q?=E5=90=8C=E5=AD=97=E6=AE=B5=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/InsertSqlParser.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index 8775c5244..d5e6b21bc 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -21,16 +21,10 @@ package com.dtstack.flink.sql.parser; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlJoin; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlMatchRecognize; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlOrderBy; -import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; import org.apache.flink.table.calcite.FlinkPlannerImpl; @@ -81,10 +75,21 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ parseNode(sqlSource, sqlParseResult); break; case SELECT: - SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); - if(sqlFrom.getKind() == IDENTIFIER){ + SqlSelect sqlSelect = (SqlSelect) sqlNode; + SqlNodeList selectList = sqlSelect.getSelectList(); + SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); + for (int index = 0; index < selectList.size(); index++) { + if (selectList.get(index).getKind().equals(SqlKind.AS)) { + sqlNodes.add(selectList.get(index)); + continue; + } + sqlNodes.add(transformToSqlBasicCall(selectList.get(index))); + } + sqlSelect.setSelectList(sqlNodes); + SqlNode sqlFrom = ((SqlSelect) sqlNode).getFrom(); + if (sqlFrom.getKind() == IDENTIFIER) { sqlParseResult.addSourceTable(sqlFrom.toString()); - }else{ + } else { parseNode(sqlFrom, sqlParseResult); } break; @@ -141,6 +146,20 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ } } + // 将 sqlNode 转换为 SqlBasicCall + public static SqlBasicCall transformToSqlBasicCall(SqlNode sqlNode) { + String asName = ""; + SqlParserPos pos = new SqlParserPos(sqlNode.getParserPosition().getLineNum(), + sqlNode.getParserPosition().getEndColumnNum()); + if (sqlNode.getKind().equals(SqlKind.IDENTIFIER)) { + asName = ((SqlIdentifier) sqlNode).names.get(1); + } + SqlNode[] operands = new SqlNode[2]; + operands[0] = sqlNode; + operands[1] = new SqlIdentifier(asName, null, pos); + return new SqlBasicCall(new SqlAsOperator(), operands, pos); + } + public static class SqlParseResult { private List sourceTableList = Lists.newArrayList(); From 1485cebca7ad34119e7e67d4d7913000a8d46f3c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 19 Jun 2020 21:55:21 +0800 Subject: [PATCH 243/378] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=97=A0=E5=85=B3imp?= =?UTF-8?q?ort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/parser/InsertSqlParser.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index d5e6b21bc..542a1f4bb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -20,10 +20,17 @@ package com.dtstack.flink.sql.parser; -import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.lang3.StringUtils; import com.google.common.collect.Lists; From eb5a0c8d2ce8e378bd5fa49e0f494a9b20cde090 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 22 Jun 2020 10:20:15 +0800 Subject: [PATCH 244/378] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E9=98=9F?= =?UTF-8?q?=E5=88=97=E6=BA=A2=E5=87=BA=E4=BF=9D=E6=8A=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/async/RdbAsyncReqRow.java | 58 ++++++++++--------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..518f457a6 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -121,35 +121,41 @@ private void connectWithRetry(Map inputParams, CRow input, Resul AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); while(!finishFlag.get()){ - CountDownLatch latch = new CountDownLatch(1); - rdbSqlClient.getConnection(conn -> { - try { - if(conn.failed()){ - connectionStatus.set(false); - if(failCounter.getAndIncrement() % 1000 == 0){ - LOG.error("getConnection error", conn.cause()); - } - if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ - resultFuture.completeExceptionally(conn.cause()); - finishFlag.set(true); + try{ + CountDownLatch latch = new CountDownLatch(1); + rdbSqlClient.getConnection(conn -> { + try { + if(conn.failed()){ + connectionStatus.set(false); + if(failCounter.getAndIncrement() % 1000 == 0){ + LOG.error("getConnection error", conn.cause()); + } + if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ + resultFuture.completeExceptionally(conn.cause()); + finishFlag.set(true); + } + return; } - return; + connectionStatus.set(true); + ScheduledFuture timerFuture = registerTimer(input, resultFuture); + cancelTimerWhenComplete(resultFuture, timerFuture); + handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); + } catch (Exception e) { + dealFillDataError(input, resultFuture, e); + } finally { + latch.countDown(); } - connectionStatus.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), inputParams, input, resultFuture); - finishFlag.set(true); - } catch (Exception e) { - dealFillDataError(input, resultFuture, e); - } finally { - latch.countDown(); + }); + try { + latch.await(); + } catch (InterruptedException e) { + LOG.error("", e); } - }); - try { - latch.await(); - } catch (InterruptedException e) { - LOG.error("", e); + + } catch (Exception e){ + //数据源队列溢出情况 + connectionStatus.set(false); } if(!finishFlag.get()){ try { From 6a35aab88637428455882b35bc35c0add423cb46 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 22 Jun 2020 14:59:59 +0800 Subject: [PATCH 245/378] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A0=87=E8=AF=86?= =?UTF-8?q?=E5=8F=98=E9=87=8FparentIsInsert=EF=BC=8C=E5=B9=B6=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E4=BB=A3=E7=A0=81=EF=BC=8C=E4=BF=AE=E6=94=B9=E5=91=BD?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/InsertSqlParser.java | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index 542a1f4bb..e9d8cc179 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -50,6 +50,9 @@ public class InsertSqlParser implements IParser { + // 用来标识当前解析节点的上一层节点是否为 insert 节点 + private static Boolean parentIsInsert = false; + @Override public boolean verify(String sql) { return StringUtils.isNotBlank(sql) && sql.trim().toLowerCase().startsWith("insert"); @@ -79,24 +82,19 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ SqlNode sqlTarget = ((SqlInsert)sqlNode).getTargetTable(); SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); sqlParseResult.addTargetTable(sqlTarget.toString()); + parentIsInsert = true; parseNode(sqlSource, sqlParseResult); break; case SELECT: SqlSelect sqlSelect = (SqlSelect) sqlNode; - SqlNodeList selectList = sqlSelect.getSelectList(); - SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); - for (int index = 0; index < selectList.size(); index++) { - if (selectList.get(index).getKind().equals(SqlKind.AS)) { - sqlNodes.add(selectList.get(index)); - continue; - } - sqlNodes.add(transformToSqlBasicCall(selectList.get(index))); + if (parentIsInsert) { + rebuildSelectNode(sqlSelect.getSelectList(), sqlSelect); } - sqlSelect.setSelectList(sqlNodes); SqlNode sqlFrom = ((SqlSelect) sqlNode).getFrom(); if (sqlFrom.getKind() == IDENTIFIER) { sqlParseResult.addSourceTable(sqlFrom.toString()); } else { + parentIsInsert = false; parseNode(sqlFrom, sqlParseResult); } break; @@ -153,8 +151,30 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ } } - // 将 sqlNode 转换为 SqlBasicCall - public static SqlBasicCall transformToSqlBasicCall(SqlNode sqlNode) { + /** + * 将第一层 select 中的 sqlNode 转化为 AsNode，解决字段名冲突问题 + * @param selectList select Node 的 select 字段 + * @param sqlSelect 第一层解析出来的 selectNode + */ + private static void rebuildSelectNode(SqlNodeList selectList, SqlSelect sqlSelect) { + SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); + + for (int index = 0; index < selectList.size(); index++) { + if (selectList.get(index).getKind().equals(SqlKind.AS)) { + sqlNodes.add(selectList.get(index)); + continue; + } + sqlNodes.add(transformToAsNode(selectList.get(index))); + } + sqlSelect.setSelectList(sqlNodes); + } + + /** + * 将 sqlNode 转化为 AsNode + * @param sqlNode 需要转化的 sqlNode + * @return 重新构造的 AsNode + */ + public static SqlBasicCall transformToAsNode(SqlNode sqlNode) { String asName = ""; SqlParserPos pos = new SqlParserPos(sqlNode.getParserPosition().getLineNum(), sqlNode.getParserPosition().getEndColumnNum()); From b1b56ad37bf5700c3e24d5442fdb7f103f68b338 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 22 Jun 2020 16:29:07 +0800 Subject: [PATCH 246/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E7=BA=BF=E7=A8=8B=E6=B1=A0=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 094f5a6ea..ae8c75f7e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -22,6 +22,7 @@ import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.enums.ECacheType; +import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.metric.MetricConstant; import com.dtstack.flink.sql.side.cache.AbstractSideCache; import com.dtstack.flink.sql.side.cache.CacheObj; @@ -68,6 +69,7 @@ public abstract class BaseAsyncReqRow extends RichAsyncFunction impl private int timeOutNum = 0; protected BaseSideInfo sideInfo; protected transient Counter parseErrorRecords; + private transient ThreadPoolExecutor cancelExecutor; public BaseAsyncReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; @@ -82,6 +84,8 @@ public void open(Configuration parameters) throws Exception { super.open(parameters); initCache(); initMetric(); + cancelExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(100000), + new DTThreadFactory("cancel-timer-executor")); LOG.info("async dim table config info: {} ", sideInfo.getSideTableInfo().toString()); } @@ -248,12 +252,11 @@ public void onProcessingTime(long timestamp) throws Exception { } protected void cancelTimerWhenComplete(ResultFuture resultFuture, ScheduledFuture timerFuture){ - ThreadPoolExecutor executors = new ThreadPoolExecutor(1, 1,0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()); if(resultFuture instanceof StreamRecordQueueEntry){ StreamRecordQueueEntry streamRecordBufferEntry = (StreamRecordQueueEntry) resultFuture; streamRecordBufferEntry.onComplete((Object value) -> { timerFuture.cancel(true); - },executors); + }, cancelExecutor); } } From 16398a1b1ffeac77408b775ac05552e0f2c40114 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 22 Jun 2020 20:08:48 +0800 Subject: [PATCH 247/378] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=B1=A0=EF=BC=8C=E8=B0=83=E6=95=B4=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/clickhouse/ClickhouseAsyncReqRow.java | 2 -- .../dtstack/flink/sql/side/db2/Db2AsyncReqRow.java | 3 --- .../flink/sql/side/impala/ImpalaAsyncReqRow.java | 2 -- .../flink/sql/launcher/PluginLoadModeTest.java | 2 +- .../flink/sql/side/mysql/MysqlAsyncReqRow.java | 2 -- .../flink/sql/side/oracle/OracleAsyncReqRow.java | 2 -- .../flink/sql/side/polardb/PolardbAsyncReqRow.java | 2 -- .../sql/side/postgresql/PostgresqlAsyncReqRow.java | 2 -- .../flink/sql/side/rdb/async/RdbAsyncReqRow.java | 14 ++++++++++---- .../sql/side/sqlserver/SqlserverAsyncReqRow.java | 2 -- 10 files changed, 11 insertions(+), 22 deletions(-) diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java index d251bf98b..7a10bf264 100644 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java @@ -67,8 +67,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("clickhouseAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java index e7ae0b650..3ebc3ac8f 100644 --- a/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java +++ b/db2/db2-side/db2-async-side/src/main/java/com/dtstack/flink/sql/side/db2/Db2AsyncReqRow.java @@ -76,9 +76,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, db2lientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("dbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); - } } diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 51fe6abc5..3f4817580 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -83,8 +83,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, impalaClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("impalaAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java index 9c4606e98..fcfc9afce 100644 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java @@ -50,7 +50,7 @@ public static void testClasspathMode() throws Exception { public static void testRocSql() throws Exception{ - String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/zy_sql/hbase_side.sql", "-name", "roc", + String[] sql = new String[]{"-mode", "local", "-sql", "/Users/roc/Documents/flink_sql/sql/impala_sink.sql", "-name", "roc", "-localSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", "-remoteSqlPluginPath", "/Users/roc/workspace/git_code/flinkStreamSQL/plugins", "-flinkconf", "/Users/roc/Documents/flink_sql/flinkconf", diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index ffdd09b51..c3b9a6a16 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -76,8 +76,6 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("mysqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index b60a794b1..e1de7b80f 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -68,7 +68,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("oracleAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java index 333b9e7e2..d29ba71b9 100644 --- a/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java +++ b/polardb/polardb-side/polardb-async-side/src/main/java/com/dtstack/flink/sql/side/polardb/PolardbAsyncReqRow.java @@ -75,7 +75,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("polardbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 3bc745e4c..1ded64371 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -76,8 +76,6 @@ public void open(Configuration parameters) throws Exception { vo.setWorkerPoolSize(rdbSideTableInfo.getAsyncPoolSize()); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, pgClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("postgresqlAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index 2374e4d2a..249b47c4f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -35,6 +35,7 @@ import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; @@ -86,6 +87,15 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { private transient ThreadPoolExecutor executor; + private final static int MAX_TASK_QUEUE_SIZE = 100000; + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + executor = new ThreadPoolExecutor(MAX_DB_CONN_POOL_SIZE_LIMIT, MAX_DB_CONN_POOL_SIZE_LIMIT, 0, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(MAX_TASK_QUEUE_SIZE), new DTThreadFactory("rdbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy()); + } + public RdbAsyncReqRow(BaseSideInfo sideInfo) { super(sideInfo); init(sideInfo); @@ -248,10 +258,6 @@ public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } - public void setExecutor(ThreadPoolExecutor executor) { - this.executor = executor; - } - private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture){ String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index aadf2b811..c96a4af28 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -78,7 +78,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("sqlServerAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From b2d9e013baa1f6bc14d7fa1b10c76645b531cc73 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 23 Jun 2020 10:03:26 +0800 Subject: [PATCH 248/378] fix tidb --- .../java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index 643b6368d..ab9269d5a 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -72,7 +72,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 8521aecf4b86fc5562393b30452545f86bfd1377 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 23 Jun 2020 10:09:07 +0800 Subject: [PATCH 249/378] fix compile error --- .../java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index 643b6368d..ab9269d5a 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -72,7 +72,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 31f1ea14a44dc5df28823b0897609031044a15b1 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 23 Jun 2020 11:52:01 +0800 Subject: [PATCH 250/378] fix compilie --- .../java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java index 643b6368d..ab9269d5a 100644 --- a/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java +++ b/tidb/tidb-side/tidb-async-side/src/main/java/com/dtstack/flink/sql/side/tidb/TidbAsyncReqRow.java @@ -72,7 +72,5 @@ public void open(Configuration parameters) throws Exception { vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSqlClient(JDBCClient.createNonShared(vertx, tidbClientConfig)); - setExecutor(new ThreadPoolExecutor(50, 50, 0, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(10000), new DTThreadFactory("TidbAsyncExec"), new ThreadPoolExecutor.CallerRunsPolicy())); } } From 8a3d6efc5fac41a52e5994b5d8a3a3fc45774ef8 Mon Sep 17 00:00:00 2001 From: gkd Date: Tue, 23 Jun 2020 15:03:56 +0800 Subject: [PATCH 251/378] =?UTF-8?q?rdb=20float=E6=95=B0=E6=8D=AE=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E8=BD=AC=E6=8D=A2=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java | 2 +- .../com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index a41cad5ef..3813386bc 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -106,7 +106,7 @@ public static void setField(PreparedStatement upload, int type, Object field, in break; case java.sql.Types.FLOAT: case java.sql.Types.DOUBLE: - upload.setDouble(index + 1, (double) field); + upload.setDouble(index + 1, Double.parseDouble(field.toString())); break; case java.sql.Types.DECIMAL: case java.sql.Types.NUMERIC: diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 3559d4376..7c3ff4b09 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -118,11 +118,11 @@ public void executeUpdate(Connection connection) { } catch (SQLException e1) { throw new RuntimeException(e1); } - metricOutputFormat.outDirtyRecords.inc(); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); LOG.error("", e); } + metricOutputFormat.outDirtyRecords.inc(); } }); rows.clear(); From 81b35d39d7162356bb7dd61abb734fe8a538f356 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 24 Jun 2020 10:35:05 +0800 Subject: [PATCH 252/378] fix redmine-27342 --- .../flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 082c546e9..23b7b19d4 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -93,7 +93,6 @@ protected void initCache() throws SQLException { protected void reloadCache() { //reload cacheRef and replace to old cacheRef Map>> newCache = Maps.newConcurrentMap(); - cacheRef.set(newCache); try { loadData(newCache); } catch (SQLException e) { @@ -123,6 +122,11 @@ public void flatMap(CRow value, Collector out) throws Exception { List> cacheList = cacheRef.get().get(cacheKey); if (CollectionUtils.isEmpty(cacheList) && sideInfo.getJoinType() == JoinType.LEFT) { out.collect(new CRow(fillData(value.row(), null), value.change())); + return; + } + + if (CollectionUtils.isEmpty(cacheList)) { + return; } cacheList.forEach(one -> out.collect(new CRow(fillData(value.row(), one), value.change()))); From 3a3bf1c909e463e45a483c6b5241f9ab5aee38ea Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 24 Jun 2020 14:58:11 +0800 Subject: [PATCH 253/378] fix redmine 27343 --- .../java/com/dtstack/flink/sql/parser/InsertSqlParser.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index e9d8cc179..2cacd01d4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -16,7 +16,6 @@ * limitations under the License. */ - package com.dtstack.flink.sql.parser; @@ -153,6 +152,7 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ /** * 将第一层 select 中的 sqlNode 转化为 AsNode，解决字段名冲突问题 + * 仅对 table.xx 这种类型的字段进行替换 * @param selectList select Node 的 select 字段 * @param sqlSelect 第一层解析出来的 selectNode */ @@ -160,7 +160,8 @@ private static void rebuildSelectNode(SqlNodeList selectList, SqlSelect sqlSelec SqlNodeList sqlNodes = new SqlNodeList(selectList.getParserPosition()); for (int index = 0; index < selectList.size(); index++) { - if (selectList.get(index).getKind().equals(SqlKind.AS)) { + if (selectList.get(index).getKind().equals(SqlKind.AS) + || ((SqlIdentifier) selectList.get(index)).names.size() == 1) { sqlNodes.add(selectList.get(index)); continue; } From 6aee24568e832c329ece0f0d60391ccabb29a733 Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 28 Jun 2020 15:07:22 +0800 Subject: [PATCH 254/378] hbase sink by dateType --- .../sql/sink/hbase/HbaseOutputFormat.java | 2 +- .../flink/sql/sink/hbase/HbaseUtil.java | 33 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index e88324936..2bebb8c01 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -129,7 +129,7 @@ private Put getPutByRow(Row record) { if (fieldVal == null) { continue; } - byte[] val = fieldVal.toString().getBytes(); + byte[] val = HbaseUtil.toByte(fieldVal); byte[] cf = families[i].getBytes(); byte[] qualifier = qualifiers[i].getBytes(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java index 6191bd177..b5a39aeff 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseUtil.java @@ -22,9 +22,15 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; +import scala.tools.nsc.Global; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Objects; /** * Created by softfly on 17/6/30. @@ -64,9 +70,30 @@ public static TypeInformation columnTypeToTypeInformation(String type) { } - public static byte[] toByte(Object value){ - if() - return new byte[]; + public static byte[] toByte(Object value) { + if (Objects.isNull(value)) { + return new byte[]{}; + } + if (value instanceof Integer) { + return Bytes.toBytes((Integer) value); + } else if (value instanceof Boolean) { + return Bytes.toBytes((Boolean) value); + } else if (value instanceof ByteBuffer) { + return Bytes.toBytes((ByteBuffer) value); + } else if (value instanceof Double) { + return Bytes.toBytes((Double) value); + } else if (value instanceof Float) { + return Bytes.toBytes((Float) value); + } else if (value instanceof Long) { + return Bytes.toBytes((Long) value); + } else if (value instanceof Short) { + return Bytes.toBytes((Short) value); + } else if (value instanceof String) { + return Bytes.toBytes(String.valueOf(value)); + } else if (value instanceof BigDecimal) { + return Bytes.toBytes((BigDecimal) value); + } + throw new RuntimeException("unkown dateType[" + value.toString() + "]"); } } From 37d25889846d0e53654e43ab0007ddbd83d862ca Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 29 Jun 2020 09:55:14 +0800 Subject: [PATCH 255/378] fix conflict --- .../flink/sql/side/hbase/HbaseAllReqRow.java | 14 +++----------- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 4 ---- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 553ffbd4b..49801a876 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -25,13 +25,12 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; -import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; +import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.map.HashedMap; -import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -39,20 +38,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; import java.sql.SQLException; diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index c56e73f03..6e8bc0783 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.sink.hbase; -import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -31,10 +30,8 @@ import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +39,6 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; -import java.util.List; import java.util.Map; import java.util.Set; From 1b39d51dcf132d87948a94f0494f4537ad5bb09a Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 29 Jun 2020 10:07:13 +0800 Subject: [PATCH 256/378] valid keytab exist --- .../com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index 57c63d243..270eb0688 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -19,10 +19,12 @@ package com.dtstack.flink.sql.sink.hbase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; /** @@ -69,6 +71,10 @@ public static UserGroupInformation loginAndReturnUGI(Configuration conf, String throw new IllegalArgumentException("keytab can not be null"); } + if (!new File(keytab).exists()){ + throw new IllegalArgumentIOException("keytab ["+ keytab + "] not exist"); + } + conf.set("hadoop.security.authentication", "Kerberos"); UserGroupInformation.setConfiguration(conf); From 9c137766e705107a13a6ca50c0aac052d3e5d3ca Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 2 Jul 2020 21:30:08 +0800 Subject: [PATCH 257/378] =?UTF-8?q?Kerberos=E8=AE=A4=E8=AF=81=E7=9A=84hbas?= =?UTF-8?q?e=20=E7=BB=B4=E8=A1=A8=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/hbase/HbaseAsyncReqRow.java | 19 +++++++++++-------- .../side/hbase/utils/HbaseConfigUtils.java | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 62524e7ed..0250decae 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -20,20 +20,16 @@ package com.dtstack.flink.sql.side.hbase; -import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; -import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.hbase.rowkeydealer.AbstractRowKeyModeDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; -import com.dtstack.flink.sql.util.AuthUtil; -import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -42,15 +38,14 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.hbase.async.Config; import org.hbase.async.HBaseClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.security.krb5.KrbException; -import java.io.File; -import java.io.IOException; import java.sql.Timestamp; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -118,9 +113,10 @@ public void open(Configuration parameters) throws Exception { if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); + System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + refreshConfig(); } - hBaseClient = new HBaseClient(config, executorService); try { @@ -148,6 +144,13 @@ public void open(Configuration parameters) throws Exception { } } + private void refreshConfig() throws KrbException { + sun.security.krb5.Config.refresh(); + KerberosName.resetDefaultRealm(); + //reload java.security.auth.login.config + javax.security.auth.login.Configuration.setConfiguration(null); + } + @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { rowKeyMode.asyncGetData(tableName, buildCacheKey(inputParams), input, resultFuture, sideInfo.getSideCache()); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index a7708aaae..c6399e27a 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -59,7 +59,7 @@ public class HbaseConfigUtils { private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; - private static final String KEY_KEY_TAB = "hbase.keytab"; + public static final String KEY_KEY_TAB = "hbase.keytab"; private static final String KEY_PRINCIPAL = "hbase.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; @@ -159,7 +159,7 @@ public static String getKeytab(Map hbaseConfigMap) { } public static void loadKrb5Conf(Map kerberosConfig) { - String krb5FilePath = MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; } @@ -183,7 +183,7 @@ public static String buildJaasStr(Map kerberosConfig) { } } - String keyTab = MapUtils.getString(kerberosConfig, KEY_KEY_TAB); + String keyTab = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_KEY_TAB); String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); StringBuilder jaasSB = new StringBuilder("Client {\n" + From 5656872f481d1420c675a5c75a3ec6159868a746 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 3 Jul 2020 15:54:42 +0800 Subject: [PATCH 258/378] fix create view with cep --- .gitignore | 1 + .../dtstack/flink/sql/parser/CreateTmpTableParser.java | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a32168f1a..d7842e1ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ target/ *.eclipse.* *.iml plugins/ +sqlplugins/ lib/ .vertx/ .DS_Store diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index 9ec5dcc0c..c0b55f7ee 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -22,11 +22,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlJoin; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import com.google.common.collect.Lists; @@ -164,6 +160,10 @@ private static void parseNode(SqlNode sqlNode, CreateTmpTableParser.SqlParserRes parseNode(unionRight, sqlParseResult); } break; + case MATCH_RECOGNIZE: + SqlMatchRecognize node = (SqlMatchRecognize) sqlNode; + sqlParseResult.addSourceTable(node.getTableRef().toString()); + break; default: //do nothing break; From e0a10435dcb243a911c0405daebc6aa667d5119d Mon Sep 17 00:00:00 2001 From: xuchao Date: Mon, 6 Jul 2020 15:43:27 +0800 Subject: [PATCH 259/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=87=BA=E7=8E=B0=E7=9A=84=E7=B1=BB=E4=B8=A2?= =?UTF-8?q?=E5=A4=B1=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sink/clickhouse/ClickhouseDialect.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java new file mode 100644 index 000000000..29bf54798 --- /dev/null +++ b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseDialect.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.clickhouse; + +import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; + +import java.util.Optional; + +/** + * Date: 2020/1/15 + * Company: www.dtstack.com + * @author maqi + */ +public class ClickhouseDialect implements JDBCDialect { + + @Override + public boolean canHandle(String url) { + return url.startsWith("jdbc:clickhouse:"); + } + + @Override + public Optional defaultDriverName() { + return Optional.of("ru.yandex.clickhouse.ClickHouseDriver"); + } + + @Override + public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { + throw new RuntimeException("Clickhouse does not support update sql, please remove primary key or use append mode"); + } +} From 603d5bcda7e817386048c057540b57007b516e2e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 8 Jul 2020 11:32:22 +0800 Subject: [PATCH 260/378] =?UTF-8?q?=E3=80=90feat=E3=80=91tableSource=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9=20charset=20=E7=9A=84=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 1 - .../DtNestRowDeserializationSchema.java | 25 +++++++++++-------- .../kafka/table/KafkaSourceTableInfo.java | 24 ++++++++++++------ 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 0af1331f3..6f2fbd9ea 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -38,7 +38,6 @@ import org.apache.flink.types.Row; import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.constrant.ConfigConstrant; import com.dtstack.flink.sql.enums.ClusterMode; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.EPluginLoadMode; diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index cf76eb903..010a5491d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -43,7 +43,7 @@ /** * source data parse to json format - * + *

* Date: 2019/12/12 * Company: www.dtstack.com * @@ -58,9 +58,12 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; + private List fieldExtraInfos; + private String charsetName; - public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos) { + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, + List fieldExtraInfos, + String charsetName) { this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); this.rowAndFieldMapping = rowAndFieldMapping; @@ -99,7 +102,7 @@ public Row deserialize(byte[] message) throws IOException { } } - private void parseTree(JsonNode jsonNode, String prefix){ + private void parseTree(JsonNode jsonNode, String prefix) { if (jsonNode.isArray()) { ArrayNode array = (ArrayNode) jsonNode; for (int i = 0; i < array.size(); i++) { @@ -118,15 +121,15 @@ private void parseTree(JsonNode jsonNode, String prefix){ return; } Iterator iterator = jsonNode.fieldNames(); - while (iterator.hasNext()){ + while (iterator.hasNext()) { String next = iterator.next(); JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); nodeAndJsonNodeMapping.put(nodeKey, child); - if(child.isArray()){ + if (child.isArray()) { parseTree(child, nodeKey); - }else { + } else { parseTree(child, nodeKey); } } @@ -137,8 +140,8 @@ private JsonNode getIgnoreCase(String key) { return nodeAndJsonNodeMapping.get(nodeMappingKey); } - private String getNodeKey(String prefix, String nodeName){ - if(Strings.isNullOrEmpty(prefix)){ + private String getNodeKey(String prefix, String nodeName) { + if (Strings.isNullOrEmpty(prefix)) { return nodeName; } return prefix + "." + nodeName; @@ -162,7 +165,7 @@ private Object convert(JsonNode node, TypeInformation info) { } else { return node.asText(); } - } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { + } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { return Date.valueOf(node.asText()); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { // local zone @@ -170,7 +173,7 @@ private Object convert(JsonNode node, TypeInformation info) { } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone return Timestamp.valueOf(node.asText()); - } else { + } else { // for types that were specified without JSON schema // e.g. POJOs try { diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 62453166f..88e3dcfc3 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -19,6 +19,7 @@ package com.dtstack.flink.sql.source.kafka.table; +import com.dtstack.flink.sql.format.FormatType; import com.dtstack.flink.sql.table.AbstractSourceTableInfo; import com.google.common.base.Preconditions; @@ -71,6 +72,7 @@ public class KafkaSourceTableInfo extends AbstractSourceTableInfo { public Map kafkaParams = new HashMap<>(); + public String charsetName; public String getBootstrapServers() { return bootstrapServers; @@ -148,11 +150,19 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } - @Override - public boolean check() { - Preconditions.checkNotNull(getType(), "kafka of type is required"); - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } + public String getCharsetName() { + return charsetName; + } + + public void setCharsetName(String charsetName) { + this.charsetName = charsetName; + } + + @Override + public boolean check() { + Preconditions.checkNotNull(getType(), "kafka of type is required"); + Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); + Preconditions.checkNotNull(topic, "kafka of topic is required"); + return false; + } } From 3e1f7a180a743bf181b4b8c962f30c0c80f128ad Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 8 Jul 2020 11:47:12 +0800 Subject: [PATCH 261/378] =?UTF-8?q?=E3=80=90fix=E3=80=91add=20charsetName?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/format/dtnest/DtNestRowDeserializationSchema.java | 8 ++++---- .../flink/sql/source/kafka/table/KafkaSourceParser.java | 1 + .../sql/source/kafka/table/KafkaSourceTableInfo.java | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 010a5491d..8f2658fdf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -53,13 +53,13 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final ObjectMapper objectMapper = new ObjectMapper(); - private Map rowAndFieldMapping; - private Map nodeAndJsonNodeMapping = Maps.newHashMap(); + private final Map rowAndFieldMapping; + private final Map nodeAndJsonNodeMapping = Maps.newHashMap(); private final String[] fieldNames; private final TypeInformation[] fieldTypes; - private List fieldExtraInfos; - private String charsetName; + private final List fieldExtraInfos; + private final String charsetName; public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos, diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index bef86f10d..cf9a75ea5 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -53,6 +53,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Thu, 9 Jul 2020 14:23:48 +0800 Subject: [PATCH 262/378] =?UTF-8?q?hbase=20=E5=85=81=E8=AE=B8=E5=A1=AB?= =?UTF-8?q?=E5=85=85null=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index cddef858e..b21f5b057 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -213,10 +213,10 @@ private Put getPutByRow(Row record) { Put put = new Put(rowKey.getBytes()); for (int i = 0; i < record.getArity(); ++i) { Object fieldVal = record.getField(i); - if (fieldVal == null) { - continue; + byte[] val = null; + if (fieldVal != null) { + val = fieldVal.toString().getBytes(); } - byte[] val = fieldVal.toString().getBytes(); byte[] cf = families[i].getBytes(); byte[] qualifier = qualifiers[i].getBytes(); From 93fb59f56786569096054d56e991ee797242ee5f Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 9 Jul 2020 15:03:37 +0800 Subject: [PATCH 263/378] =?UTF-8?q?hbase=20=E5=88=97=E6=97=8F=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F=E9=94=99=E4=B9=B1=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java index 5105e0fc0..35fe16941 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseSinkParser.java @@ -129,7 +129,7 @@ public void parseFieldsInfo(String fieldsInfo, HbaseTableInfo tableInfo){ } private Map parseColumnFamily(Map physicalFieldMap){ - Map columnFamiles = Maps.newHashMap(); + Map columnFamiles = new LinkedHashMap<>(); physicalFieldMap.values().forEach(x -> { String[] columnFamily = StringUtils.split(x.trim(), ":"); columnFamiles.put(x, columnFamily[1]); From 4c3cd67908c5362cb38b854a7b1118fb1deddd7d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 9 Jul 2020 19:23:44 +0800 Subject: [PATCH 264/378] =?UTF-8?q?=E3=80=90fix=E3=80=91fix=20charset=20de?= =?UTF-8?q?code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/format/dtnest/DtNestRowDeserializationSchema.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 8f2658fdf..d6e26538f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -34,6 +34,7 @@ import org.apache.flink.types.Row; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -73,7 +74,8 @@ public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map Date: Thu, 9 Jul 2020 20:31:52 +0800 Subject: [PATCH 265/378] =?UTF-8?q?=E3=80=90fix=E3=80=91decode=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/format/dtnest/DtNestRowDeserializationSchema.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index d6e26538f..68f7a2f87 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -74,8 +74,7 @@ public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map Date: Thu, 9 Jul 2020 21:29:34 +0800 Subject: [PATCH 266/378] =?UTF-8?q?=E6=96=B0=E5=A2=9Eupdate=E8=AF=AD?= =?UTF-8?q?=E5=8F=A5=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/impala/ImpalaDialect.java | 23 ++++++++++---- .../flink/sql/sink/impala/ImpalaSink.java | 2 +- .../sql/sink/rdb/JDBCTypeConvertUtils.java | 31 ++++++++++++++++++- .../sink/rdb/writer/AbstractUpsertWriter.java | 4 ++- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java index 89fb769c3..754be49d6 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaDialect.java @@ -20,13 +20,11 @@ import com.dtstack.flink.sql.sink.rdb.dialect.JDBCDialect; import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; /** @@ -41,8 +39,11 @@ public class ImpalaDialect implements JDBCDialect { private TypeInformation[] fieldTypes; - public ImpalaDialect(TypeInformation[] fieldTypes){ + private List primaryKeys; + + public ImpalaDialect(TypeInformation[] fieldTypes, List primaryKeys){ this.fieldTypes = fieldTypes; + this.primaryKeys = primaryKeys; } @Override @@ -62,7 +63,17 @@ public String quoteIdentifier(String identifier) { @Override public String getUpdateStatement(String tableName, String[] fieldNames, String[] conditionFields) { - throw new RuntimeException("impala does not support update sql, please remove primary key or use append mode"); + //跳过primary key字段 + String setClause = Arrays.stream(fieldNames) + .filter(f -> CollectionUtils.isNotEmpty(primaryKeys) ? !primaryKeys.contains(f) : true) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); + String conditionClause = Arrays.stream(conditionFields) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + return "UPDATE " + quoteIdentifier(tableName) + + " SET " + setClause + + " WHERE " + conditionClause; } @Override diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 910f6f4c5..34101d410 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -48,7 +48,7 @@ public ImpalaSink() { public JDBCUpsertOutputFormat getOutputFormat() { JDBCOptions jdbcOptions = JDBCOptions.builder() .setDbUrl(getImpalaJdbcUrl()) - .setDialect(new ImpalaDialect(getFieldTypes())) + .setDialect(new ImpalaDialect(getFieldTypes(), primaryKeys)) .setUsername(userName) .setPassword(password) .setTableName(tableName) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index 3813386bc..ded966c68 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.rdb; +import com.google.common.collect.Lists; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,10 @@ public class JDBCTypeConvertUtils { * @see PreparedStatement */ public static void setRecordToStatement(PreparedStatement upload, int[] typesArray, Row row) throws SQLException { + setRecordToStatement(upload, typesArray, row, null); + } + + public static void setRecordToStatement(PreparedStatement upload, int[] typesArray, Row row, int[] pkFieldIndices) throws SQLException { if (typesArray != null && typesArray.length > 0 && typesArray.length != row.getArity()) { LOG.warn("Column SQL types array doesn't match arity of passed Row! Check the passed array..."); } @@ -62,10 +67,34 @@ public static void setRecordToStatement(PreparedStatement upload, int[] typesArr } } else { // types provided + int placeIndex = 0; for (int i = 0; i < row.getArity(); i++) { - setField(upload, typesArray[i], row.getField(i), i); + if(isPrimaryKeyField(pkFieldIndices, i)){ + continue; + } + setField(upload, typesArray[i], row.getField(i), placeIndex ++); + } + + //填充whereClause中的主键占位符 + if (pkFieldIndices != null && pkFieldIndices.length > 0) { + for (int j = 0; j < pkFieldIndices.length; j++) { + int pkIndex = pkFieldIndices[j]; + setField(upload, typesArray[pkIndex], row.getField(pkIndex), placeIndex + j); + } + } + } + } + + private static boolean isPrimaryKeyField(int[] pkFieldIndices, int fieldIndex) { + if (pkFieldIndices == null || pkFieldIndices.length <= 0) { + return false; + } + for (int index : pkFieldIndices) { + if (index == fieldIndex) { + return true; } } + return false; } public static void setField(PreparedStatement upload, int type, Object field, int index) throws SQLException { diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 2c070b680..42ed545a6 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -268,6 +268,7 @@ private static final class UpsertWriterUsingInsertUpdateStatement extends Abstra private final String existSql; private final String insertSql; private final String updateSql; + private final int[] pkFields; private transient PreparedStatement existStatement; private transient PreparedStatement insertStatement; @@ -287,6 +288,7 @@ private UpsertWriterUsingInsertUpdateStatement( this.existSql = existSql; this.insertSql = insertSql; this.updateSql = updateSql; + this.pkFields = pkFields; } @Override @@ -310,7 +312,7 @@ void processOneRowInBatch(Row pk, Row row) throws SQLException { resultSet.close(); if (exist) { // do update - setRecordToStatement(updateStatement, fieldTypes, row); + setRecordToStatement(updateStatement, fieldTypes, row, pkFields); updateStatement.addBatch(); } else { // do insert From 27c0f8b4428c664b65492617003cca3551a4bee2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 13 Jul 2020 11:36:19 +0800 Subject: [PATCH 267/378] =?UTF-8?q?=E5=85=BC=E5=AE=B9primaryKey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/sink/redis/table/RedisSinkParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index e965eeecb..8961f7da9 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -50,11 +50,11 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { + List primaryKeysList = Lists.newArrayList(); primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); + redisTableInfo.setPrimaryKeys(primaryKeysList); } - redisTableInfo.setPrimaryKeys(primaryKeysList); return redisTableInfo; } From a8d3a3276770612436ecee1377efff8b874ed9bd Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 13 Jul 2020 14:31:29 +0800 Subject: [PATCH 268/378] add array type mode --- .gitignore | 1 + .../com/dtstack/flink/sql/util/ClassUtil.java | 10 +- .../dtstack/flink/sql/util/DataTypeUtils.java | 157 ++++++++++++++++++ .../flink/sql/util/DataTypeUtilsTest.java | 54 ++++++ .../sql/source/kafka/AbstractKafkaSource.java | 13 +- 5 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java diff --git a/.gitignore b/.gitignore index a32168f1a..d7842e1ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ target/ *.eclipse.* *.iml plugins/ +sqlplugins/ lib/ .vertx/ .DS_Store diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 4bb4ff0d8..7d092141e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.util; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.sql.Date; import java.sql.Time; @@ -34,7 +35,14 @@ public class ClassUtil { public static Class stringConvertClass(String str) { - switch (str.toLowerCase()) { + + // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型 + String lowerStr = str.toLowerCase(); + if (lowerStr.startsWith("array")) { + return Array.newInstance(Integer.class, 0).getClass(); + } + + switch (lowerStr) { case "boolean": case "bit": return Boolean.class; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java new file mode 100644 index 000000000..ba1a76fd4 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -0,0 +1,157 @@ +package com.dtstack.flink.sql.util; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.apache.flink.api.common.typeinfo.TypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.shaded.guava18.com.google.common.base.Splitter; +import org.apache.flink.table.api.Types; +import scala.reflect.macros.TypecheckException; + +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @program: flink.sql + * @author: wuren + * @create: 2020/07/09 + **/ +public class DataTypeUtils { + + private final static Pattern COMPOUND_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); + private final static String ARRAY = "ARRAY"; + private final static String ROW = "ROW"; + private final static char FIELD_DELIMITER = ','; + private final static char TYPE_DELIMITER = ' '; + + private DataTypeUtils() {} + + public static TypeInformation convertToCollectionType(String string) { + Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + // TODO 现在只支持ARRAY类型后续可以加入 MAP等类型 + if (matcher.find() && ARRAY.equals(matcher.group(1))) { + return convertToArray(string); + } else { + throw new RuntimeException(""); + } + } + + /** + * 目前ARRAY里只支持ROW和其他基本类型 + * @param arrayTypeString + * @return + */ + public static TypeInformation convertToArray(String arrayTypeString) { + Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(arrayTypeString); + if (matcher.find() && ARRAY.equals(matcher.group(1))) { + String elementTypeString = matcher.group(2); + TypeInformation elementType; + if (elementTypeString.toUpperCase().startsWith(ROW)) { + elementType = convertToRow(elementTypeString); + } else { + elementType = convertToAtomicType(elementTypeString); + } + return Types.OBJECT_ARRAY(elementType); + } else { + throw new RuntimeException("convert to array type error!"); + } + + } + + /** + * 目前ROW里只支持基本类型 + * @param string + */ + public static RowTypeInfo convertToRow(String string) { + Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + + if (matcher.find() && + ROW.equals( + matcher.group(1).toUpperCase() + ) + ) { + String elementTypeStr = matcher.group(2); + Iterable typeInfo = splitCompositeTypeFields(elementTypeStr); + Tuple2 tuple = genFieldInfo(typeInfo); + return new RowTypeInfo(tuple.f0, tuple.f1); + } else { + throw new RuntimeException(""); + } + } + + private static Iterable splitCompositeTypeFields(String string) { + return Splitter + .on(FIELD_DELIMITER) + .trimResults() + .split(string); + } + + private static Tuple2 genFieldInfo(Iterable typeInfo) { + int fieldsSize = Iterators.size(typeInfo.iterator()); + ArrayList types = Lists.newArrayList(); + ArrayList fieldNames = Lists.newArrayList(); + for (String type : typeInfo) { + Iterable fieldInfo = Splitter + .on(TYPE_DELIMITER) + .trimResults() + .omitEmptyStrings() + .split(type); + ArrayList array = Lists.newArrayList(fieldInfo.iterator()); + if (array.size() == 2) { + TypeInformation fieldType = convertToAtomicType(array.get(1)); + types.add(fieldType); + fieldNames.add(array.get(0)); + } else { + throw new RuntimeException(); + } + } + + TypeInformation[] typesArray = types.toArray(new TypeInformation[types.size()]); + String[] fieldNamesArray = fieldNames.toArray(new String[fieldNames.size()]); + return Tuple2.of(typesArray, fieldNamesArray); + } + + /** + * 转换基本类型，所有类型参考Flink官方文档，一共12个基本类型。 + * @param string + * @return + */ + public static TypeInformation convertToAtomicType(String string) { + switch (string.toUpperCase()) { + case "VARCHAR": + case "STRING": + return Types.STRING(); + case "BOOLEAN": + return Types.BOOLEAN(); + case "TINYINT": + return Types.BYTE(); + case "SMALLINT": + return Types.SHORT(); + case "INT": + case "INTEGER": + return Types.INT(); + case "BIGINT": + return Types.LONG(); + case "FLOAT": + case "REAL": + return Types.FLOAT(); + case "DOUBLE": + return Types.DOUBLE(); + case "DECIMAL": + case "DEC": + case "NUMERIC": + return Types.DECIMAL(); + case "DATE": + return Types.SQL_DATE(); + case "TIME": + return Types.SQL_TIME(); + case "TIMESTAMP": + return Types.SQL_TIMESTAMP(); + default: + throw new RuntimeException("type " + string + "not supported"); + } + } +} diff --git a/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java new file mode 100644 index 000000000..4d5627348 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java @@ -0,0 +1,54 @@ +package com.dtstack.flink.sql.util; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.table.api.Types; +import org.apache.flink.types.Row; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.Array; + +import static org.junit.Assert.*; + +public class DataTypeUtilsTest { + + @Test + public void convertToArray() { + String atomicStr = "ARRAY"; + String compositeTypeStr = "ARRAY>"; + + TypeInformation atomicArrayType = DataTypeUtils.convertToArray(atomicStr); + TypeInformation compositeTypeArrayType = DataTypeUtils.convertToArray(compositeTypeStr); + + + String[] normalFieldNames = new String[] {"id", "name"}; + TypeInformation[] normalTypes = new TypeInformation[] {Types.INT(), Types.STRING()}; + RowTypeInfo rowTypeInfo = new RowTypeInfo(normalTypes, normalFieldNames); + TypeInformation normalAtomicType = Types.OBJECT_ARRAY(Types.STRING()); + TypeInformation normalCompositeType = Types.OBJECT_ARRAY(rowTypeInfo); + + Assert.assertTrue(normalAtomicType.equals(atomicArrayType)); + Assert.assertTrue(normalCompositeType.equals(compositeTypeArrayType)); + } + + @Test + public void convertToRow() { + String string = "ROW"; + RowTypeInfo rowType = DataTypeUtils.convertToRow(string); + + String[] fieldNames = rowType.getFieldNames(); + Assert.assertTrue("id".equals(fieldNames[0])); + Assert.assertTrue("name".equals(fieldNames[1])); + + TypeInformation[] fieldTypes = rowType.getFieldTypes(); + Assert.assertTrue(Types.INT() == fieldTypes[0]); + Assert.assertTrue(Types.STRING() == fieldTypes[1]); + } + + @Test + public void convertToAtomicType() { + TypeInformation type = DataTypeUtils.convertToAtomicType("STRING"); + Assert.assertTrue(type == Types.STRING()); + } +} \ No newline at end of file diff --git a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java index 852a381e2..2cc94b9d6 100644 --- a/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java +++ b/kafka-base/kafka-base-source/src/main/java/com/dtstack/flink/sql/source/kafka/AbstractKafkaSource.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.source.IStreamSourceGener; import com.dtstack.flink.sql.source.kafka.enums.EKafkaOffset; import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; +import com.dtstack.flink.sql.util.DataTypeUtils; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.StringUtils; @@ -32,6 +33,7 @@ import org.apache.flink.types.Row; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.lang.reflect.Array; import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; @@ -71,9 +73,16 @@ protected String generateOperatorName(String tabName, String topicName) { } protected TypeInformation getRowTypeInformation(KafkaSourceTableInfo kafkaSourceTableInfo) { + String[] fieldTypes = kafkaSourceTableInfo.getFieldTypes(); Class[] fieldClasses = kafkaSourceTableInfo.getFieldClasses(); - TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + TypeInformation[] types = + IntStream.range(0, fieldClasses.length) + .mapToObj(i -> { + if (fieldClasses[i].isArray()) { + return DataTypeUtils.convertToArray(fieldTypes[i]); + } + return TypeInformation.of(fieldClasses[i]); + }) .toArray(TypeInformation[]::new); return new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); From 42f22145def750f4bf60faebbce06c81742875bb Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 14 Jul 2020 10:13:19 +0800 Subject: [PATCH 269/378] add field split pattern --- .../dtstack/flink/sql/table/AbstractTableParser.java | 2 +- .../com/dtstack/flink/sql/util/DataTypeUtils.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 1f210cbf2..5b344d118 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -85,7 +85,7 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ } public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ - + // TODO 替换 List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index ba1a76fd4..92e6ca40f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -2,13 +2,11 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import org.apache.flink.api.common.typeinfo.TypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.guava18.com.google.common.base.Splitter; import org.apache.flink.table.api.Types; -import scala.reflect.macros.TypecheckException; import java.util.ArrayList; import java.util.regex.Matcher; @@ -27,6 +25,11 @@ public class DataTypeUtils { private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; + private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*,"); + private final static Pattern ATOMIC_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*,"); + private final static Pattern TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*"); + private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*"); + private DataTypeUtils() {} public static TypeInformation convertToCollectionType(String string) { @@ -154,4 +157,9 @@ public static TypeInformation convertToAtomicType(String string) { throw new RuntimeException("type " + string + "not supported"); } } + + public static void splitFields(String fieldStmt) { + // TODO 可以把每轮字符流开始的空白字符去掉。 + + } } From f1eafc25724458e76af843b3c7d06b5b6069f3b7 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 14 Jul 2020 14:32:20 +0800 Subject: [PATCH 270/378] fix side table join error msg --- .gitignore | 1 + core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a32168f1a..d7842e1ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ target/ *.eclipse.* *.iml plugins/ +sqlplugins/ lib/ .vertx/ .DS_Store diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java index 7b4d3437b..4bc3ecd8f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseSideInfo.java @@ -162,7 +162,7 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName){ } } if(equalFieldIndex == -1){ - throw new RuntimeException("can't find equal field " + rightField); + throw new RuntimeException("can't find equal field " + leftField); } equalValIndex.add(equalFieldIndex); From 30e57d595025a8d2c0fa6e810ac3823c2dd96bea Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 14 Jul 2020 20:30:40 +0800 Subject: [PATCH 271/378] array type support complete --- .../flink/sql/table/AbstractTableParser.java | 38 ++++++++++++++++++- .../com/dtstack/flink/sql/util/ClassUtil.java | 1 + .../dtstack/flink/sql/util/DataTypeUtils.java | 37 +++++++++++++++--- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 5b344d118..d633c114d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -27,6 +27,8 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -50,6 +52,9 @@ public abstract class AbstractTableParser { private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<"); + private static Pattern compositeTypeTailPattern = Pattern.compile(">\\s*>"); + private Map patternMap = Maps.newHashMap(); private Map handlerMap = Maps.newHashMap(); @@ -85,8 +90,11 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ } public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ - // TODO 替换 + List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); + + ArrayList cache = new ArrayList<>(); + boolean currentIsCompositeType = false; for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); @@ -94,7 +102,33 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] filedInfoArr = fieldRow.split("\\s+"); + // 处理复合类型，例如 ARRAY> + String[] filedInfoArr; + Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); + Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); + + if (tailMatcher.find()) { + cache.add(fieldRow); + currentIsCompositeType = false; + fieldRow = String.join("", cache); + cache.clear(); + String[] tmp = fieldRow.split("\\s+"); + String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); + filedInfoArr = new String[] { + tmp[0], + String.join(" ", type) + }; + } else if (headMatcher.find() || currentIsCompositeType) { + currentIsCompositeType = true; + StringBuilder builder = new StringBuilder(); + builder.append(fieldRow); + builder.append(","); + cache.add(builder.toString()); + continue; + } else { + filedInfoArr = fieldRow.split("\\s+"); + } + if(filedInfoArr.length < 2 ){ throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 7d092141e..5fa96aca9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -69,6 +69,7 @@ public static Class stringConvertClass(String str) { case "varchar": case "char": case "text": + case "string": return String.class; case "real": diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 92e6ca40f..1602205b6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.util; +import com.google.common.base.Strings; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -9,6 +10,7 @@ import org.apache.flink.table.api.Types; import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,10 +27,10 @@ public class DataTypeUtils { private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; - private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*,"); private final static Pattern ATOMIC_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*,"); - private final static Pattern TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*"); - private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*"); + private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+?>)\\s*,"); + private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*$"); + private final static Pattern COMPLEX_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*$"); private DataTypeUtils() {} @@ -158,8 +160,33 @@ public static TypeInformation convertToAtomicType(String string) { } } - public static void splitFields(String fieldStmt) { - // TODO 可以把每轮字符流开始的空白字符去掉。 + public static ArrayList fieldStmtLexer(String fieldStmts) { + + String stmtStream = fieldStmts; + ArrayList tokens = new ArrayList<>(); + while (Strings.isNullOrEmpty(stmtStream)) { + Matcher atomicTypeMatcher = ATOMIC_TYPE_PATTERN.matcher(stmtStream); + Matcher complexTypeMatcher = COMPLEX_TYPE_PATTERN.matcher(stmtStream); + Matcher atomicTypeTailMatcher = ATOMIC_TAIL_PATTERN.matcher(stmtStream); + Matcher complexTypeTailMatcher = COMPLEX_TAIL_PATTERN.matcher(stmtStream); + + String fieldStmt; + + if (atomicTypeMatcher.find()) { + fieldStmt = atomicTypeMatcher.group(0); + } else if (complexTypeMatcher.find()) { + fieldStmt = complexTypeMatcher.group(0); + } else if (atomicTypeTailMatcher.find()) { + fieldStmt = atomicTypeTailMatcher.group(0); + } else if (complexTypeTailMatcher.find()) { + fieldStmt = complexTypeTailMatcher.group(0); + } else { + throw new RuntimeException("field declaration statement error" + fieldStmts); + } + tokens.add(fieldStmt); + stmtStream = stmtStream.substring(fieldStmt.length() + 1); + } + return tokens; } } From 82291f9870f38a32fbeb78c3fa1e550b24cb8871 Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 16 Jul 2020 14:23:51 +0800 Subject: [PATCH 272/378] add array type AS keyword support --- .../flink/sql/table/AbstractTableParser.java | 123 ++++++++++-------- .../dtstack/flink/sql/util/DataTypeUtils.java | 5 + 2 files changed, 73 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index d633c114d..beed69fcb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -48,12 +48,12 @@ public abstract class AbstractTableParser { private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<"); - private static Pattern compositeTypeTailPattern = Pattern.compile(">\\s*>"); + private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<.+"); + private static Pattern compositeTypeTailPattern = Pattern.compile(".*>\\s*>.*"); private Map patternMap = Maps.newHashMap(); @@ -93,8 +93,8 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); - ArrayList cache = new ArrayList<>(); - boolean currentIsCompositeType = false; + ArrayList buffer = new ArrayList<>(); + for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); @@ -103,65 +103,58 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ } // 处理复合类型，例如 ARRAY> - String[] filedInfoArr; Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); - - if (tailMatcher.find()) { - cache.add(fieldRow); - currentIsCompositeType = false; - fieldRow = String.join("", cache); - cache.clear(); - String[] tmp = fieldRow.split("\\s+"); - String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); - filedInfoArr = new String[] { - tmp[0], - String.join(" ", type) - }; - } else if (headMatcher.find() || currentIsCompositeType) { - currentIsCompositeType = true; - StringBuilder builder = new StringBuilder(); - builder.append(fieldRow); - builder.append(","); - cache.add(builder.toString()); - continue; + if ( + !tailMatcher.matches() && + (headMatcher.matches() || + !buffer.isEmpty()) + ) { + writeBuffer(buffer, fieldRow); } else { - filedInfoArr = fieldRow.split("\\s+"); - } - - if(filedInfoArr.length < 2 ){ - throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); - } - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); - if(isMatcherKey){ - continue; - } + String[] fieldInfoArr; + if (tailMatcher.matches()) { + buffer.add(fieldRow); + fieldRow = String.join("", buffer); + fieldInfoArr = readBuffer(buffer); + } else { + fieldInfoArr = fieldRow.split("\\s+"); + } - //Compatible situation may arise in space in the fieldName - String[] filedNameArr = new String[filedInfoArr.length - 1]; - System.arraycopy(filedInfoArr, 0, filedNameArr, 0, filedInfoArr.length - 1); - String fieldName = String.join(" ", filedNameArr); - String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); + if (fieldInfoArr.length < 2) { + throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); + } + boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); + if (isMatcherKey) { + continue; + } - Class fieldClass = null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + //Compatible situation may arise in space in the fieldName + String[] filedNameArr = new String[fieldInfoArr.length - 1]; + System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); + String fieldName = String.join(" ", filedNameArr); + String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); + + Class fieldClass = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; + + Matcher matcher = charTypePattern.matcher(fieldType); + if (matcher.find()) { + fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + } else { + fieldClass = dbTypeConvertToJavaType(fieldType); + } - Matcher matcher = charTypePattern.matcher(fieldType); - if (matcher.find()) { - fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); - } else { - fieldClass = dbTypeConvertToJavaType(fieldType); + tableInfo.addPhysicalMappings(fieldInfoArr[0],fieldInfoArr[0]); + tableInfo.addField(fieldName); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(fieldExtraInfo); } - - tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); - tableInfo.addField(fieldName); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); } tableInfo.finish(); @@ -206,4 +199,24 @@ protected void addParserHandler(String parserName, Pattern pattern, ITableFieldD patternMap.put(parserName, pattern); handlerMap.put(parserName, handler); } + + private void writeBuffer(List buffer, String fieldRow) { + StringBuilder builder = new StringBuilder(); + builder.append(fieldRow); + builder.append(","); + buffer.add(builder.toString()); + } + + private String[] readBuffer(List buffer) { + String fieldRow = String.join("", buffer); + buffer.clear(); + String[] tmp = fieldRow.split("\\s+"); + String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); + String[] fieldInfoArr = new String[] { + tmp[0], + String.join(" ", type) + }; + return fieldInfoArr; + } + } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 1602205b6..88ae95ec8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -160,6 +160,11 @@ public static TypeInformation convertToAtomicType(String string) { } } + /** + * 目前这个方法未使用，设置当初是想字段声明走统一的词法分析器(分词器)。 + * @param fieldStmts + * @return + */ public static ArrayList fieldStmtLexer(String fieldStmts) { String stmtStream = fieldStmts; From 12107d4e7b4711c06112f194a95ced7ab46ab155 Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 16 Jul 2020 17:24:31 +0800 Subject: [PATCH 273/378] DT_NEST deserialization support ObjeactArray and Row Type --- .../DtNestRowDeserializationSchema.java | 83 +++++++++++++------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 5f1b1c6f3..d3ed68945 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -24,6 +24,7 @@ import org.apache.flink.api.common.serialization.AbstractDeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; @@ -34,6 +35,7 @@ import org.apache.flink.types.Row; import java.io.IOException; +import java.lang.reflect.Array; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -71,30 +73,7 @@ public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map info) { } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone return Timestamp.valueOf(node.asText()); - } else { + } else if (info instanceof RowTypeInfo) { + return convertRow(node, (RowTypeInfo) info); + } else if (info instanceof ObjectArrayTypeInfo) { + return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); + } else { // for types that were specified without JSON schema // e.g. POJOs try { @@ -179,5 +162,55 @@ private Object convert(JsonNode node, TypeInformation info) { } } + private Row convertTopRow() { + Row row = new Row(fieldNames.length); + try { + for (int i = 0; i < fieldNames.length; i++) { + JsonNode node = getIgnoreCase(fieldNames[i]); + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); + if (node == null) { + if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { + throw new IllegalStateException("Failed to find field with name '" + + fieldNames[i] + "'."); + } else { + row.setField(i, null); + } + } else { + // Read the value as specified type + Object value = convert(node, fieldTypes[i]); + row.setField(i, value); + } + } + return row; + } finally { + nodeAndJsonNodeMapping.clear(); + } + } + + private Row convertRow(JsonNode node, RowTypeInfo info) { + final String[] names = info.getFieldNames(); + final TypeInformation[] types = info.getFieldTypes(); + + final Row row = new Row(names.length); + for (int i = 0; i < names.length; i++) { + final String name = names[i]; + final JsonNode subNode = node.get(name); + if (subNode == null) { + row.setField(i, null); + } else { + row.setField(i, convert(subNode, types[i])); + } + } + + return row; + } + + private Object convertObjectArray(JsonNode node, TypeInformation elementType) { + final Object[] array = (Object[]) Array.newInstance(elementType.getTypeClass(), node.size()); + for (int i = 0; i < node.size(); i++) { + array[i] = convert(node.get(i), elementType); + } + return array; + } } From d5c06c72287491d379ed719cc37fc1d97028e8ee Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 17 Jul 2020 11:36:29 +0800 Subject: [PATCH 274/378] fix kafka sink doc --- docs/kafkaSink.md | 223 -------------------------------------- docs/plugin/kafkaSink.md | 229 +++++++++++++++++++++++++-------------- docs/pluginsInfo.md | 1 + 3 files changed, 151 insertions(+), 302 deletions(-) delete mode 100644 docs/kafkaSink.md diff --git a/docs/kafkaSink.md b/docs/kafkaSink.md deleted file mode 100644 index 5d7c7b2a7..000000000 --- a/docs/kafkaSink.md +++ /dev/null @@ -1,223 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - function(colNameX) AS aliasName, - WATERMARK FOR colName AS withOffset( colName , delayTime ) - )WITH( - type ='kafka11', - bootstrapServers ='ip:port,ip:port...', - zookeeperQuorum ='ip:port,ip:port/zkparent', - offsetReset ='latest', - topic ='topicName', - groupId='test', - parallelism ='parllNum', - ); -``` - -## 2.支持的版本 - kafka09,kafka10,kafka11及以上版本 - **kafka读取和写入的版本必须一致，否则会有兼容性错误。** - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| -|groupId | 需要读取的 groupId 名称|否|| -|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|topic | 需要读取的 topic 名称|是|| -|parallelism | 并行度设置|否|1| -|partitionKeys | 用来分区的字段|否|| -|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| -|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| -|fieldDelimiter | csv数据分隔符|否| , | - - -**kafka相关参数可以自定义，使用kafka.开头即可。** -``` -kafka.consumer.id -kafka.socket.timeout.ms -kafka.fetch.message.max.bytes -kafka.num.consumer.fetchers -kafka.auto.commit.enable -kafka.auto.commit.interval.ms -kafka.queued.max.message.chunks -kafka.rebalance.max.retries -kafka.fetch.min.bytes -kafka.fetch.wait.max.ms -kafka.rebalance.backoff.ms -kafka.refresh.leader.backoff.ms -kafka.consumer.timeout.ms -kafka.exclude.internal.topics -kafka.partition.assignment.strategy -kafka.client.id -kafka.zookeeper.session.timeout.ms -kafka.zookeeper.connection.timeout.ms -kafka.zookeeper.sync.time.ms -kafka.offsets.storage -kafka.offsets.channel.backoff.ms -kafka.offsets.channel.socket.timeout.ms -kafka.offsets.commit.max.retries -kafka.dual.commit.enabled -kafka.partition.assignment.strategy -kafka.socket.receive.buffer.bytes -kafka.fetch.min.bytes -``` - -## 5.样例： - -### json格式： -``` -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - partitionKeys = 'channel,pv', - updateMode='upsert' - ); - -upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} -append模式下发的数据格式：{"channel":"zs","pv":"330"} - -``` - -### avro格式： - -如果updateMode='upsert',schemaInfo需要包含retract属性信息。 - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='1', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - --type='console' - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='1', - updateMode='upsert', - sinkdatatype = 'avro', - schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} - ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, - {"name":"retract","type":"boolean"}]}' - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` -### csv格式： - -``` -CREATE TABLE MyTable( - channel varchar, - pv varchar - --xctime bigint - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - groupId='mqTest01', - offsetReset='latest', - topic='mqTest01', - parallelism ='2', - topicIsPattern ='false' - ); - -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache = 'LRU', - cacheTTLMs='10000', - parallelism ='1' - - ); - - -CREATE TABLE MyResult( - channel varchar, - pv varchar - )WITH( - type='kafka', - bootstrapServers='172.16.8.107:9092', - topic='mqTest02', - parallelism ='2', - updateMode='upsert', - sinkdatatype = 'csv', - fieldDelimiter='*' - - - - ); - - -insert -into - MyResult - select - a.channel as channel, - a.pv as pv - from - MyTable a -``` diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 1d8d8ce5f..86b01f3ee 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -3,46 +3,47 @@ CREATE TABLE tableName( colName colType, ... - colNameX colType, - [primary key (colName)] + function(colNameX) AS aliasName, + WATERMARK FOR colName AS withOffset( colName , delayTime ) )WITH( - type ='kafka09', + type ='kafka11', bootstrapServers ='ip:port,ip:port...', zookeeperQuorum ='ip:port,ip:port/zkparent', offsetReset ='latest', topic ='topicName', groupId='test', parallelism ='parllNum', - timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 ); ``` -## 2.支持版本 - kafka09,kafka10,kafka11及以上版本 - +## 2.支持的版本 + kafka09,kafka10,kafka11及以上版本 + **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + ## 3.表结构定义 |参数名称|含义| |----|---| -| tableName| 结果表名称| +| tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](../colType.md)| ## 4.参数： - + |参数名称|含义|是否必填|默认值| -|----|----|----|----| -|type |表名的输出表类型[kafka09|kafka10|kafka11|kafka]|是|| +|----|---|---|---| +|type | kafka09 | 是|kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| |groupId | 需要读取的 groupId 名称|否|| |bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| |zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| |topic | 需要读取的 topic 名称|是|| -|topicIsPattern | topic是否是正则表达式格式(true|false) |否| false -|offsetReset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| -|timezone|时区设置[timezone支持的参数](../timeZone.md)|否|'Asia/Shanghai' +|partitionKeys | 用来分区的字段|否|| +|updateMode | 回溯流数据下发模式，append,upsert.upsert模式下会将是否为回溯信息以字段形式进行下发。|否|append| +|sinkdatatype | 写入kafka数据格式，json,avro,csv|否|json| +|fieldDelimiter | csv数据分隔符|否| , | + + **kafka相关参数可以自定义，使用kafka.开头即可。** ``` kafka.consumer.id @@ -74,79 +75,149 @@ kafka.socket.receive.buffer.bytes kafka.fetch.min.bytes ``` -## 5.完整样例： +## 5.样例： + +### json格式： +``` +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + partitionKeys = 'channel,pv', + updateMode='upsert' + ); + +upsert模式下发的数据格式：{"channel":"zs","pv":"330",retract:true} +append模式下发的数据格式：{"channel":"zs","pv":"330"} + +``` + +### avro格式： + +如果updateMode='upsert',schemaInfo需要包含retract属性信息。 + ``` CREATE TABLE MyTable( - id bigint, - name varchar, - address varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_in2', - groupId = 'flink_sql', - timezone = 'Asia/Shanghai', - topicIsPattern = 'false', - parallelism = '1' -); - -CREATE TABLE sideTable( - id bigint, - school varchar, - home varchar, - PRIMARY KEY(id), + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='1', + topicIsPattern ='false' + ); + +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), PERIOD FOR SYSTEM_TIME -)WITH( + )WITH( type='mysql', - url='jdbc:mysql://172.16.8.109:3306/tiezhu', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', userName='dtstack', password='abc123', - tableName='stressTest', - cache='ALL', - parallelism='1' -); + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + CREATE TABLE MyResult( - id bigint, - name varchar, - address varchar, - home varchar, - school varchar -)WITH( - type = 'kafka10', - bootstrapServers = '172.16.101.224:9092', - zookeeperQuorm = '172.16.100.188:2181/kafka', - offsetReset = 'latest', - topic = 'tiezhu_test_out2', - parallelism = '1' -); - -insert + channel varchar, + pv varchar + )WITH( + --type='console' + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='1', + updateMode='upsert', + sinkdatatype = 'avro', + schemaInfo = '{"type":"record","name":"MyResult","fields":[{"name":"channel","type":"string"} + ,{"name":"pv","type":"string"},{"name":"channel","type":"string"}, + {"name":"retract","type":"boolean"}]}' + + ); + + +insert into - MyResult + MyResult select - t1.id AS id, - t1.name AS name, - t1.address AS address, - t2.school AS school, - t2.home AS home - from( - select - id, - name, - address + a.channel as channel, + a.pv as pv from - MyTable - ) t1 - join sideTable t2 - on t1.id = t2.id; - ``` + MyTable a +``` +### csv格式： + +``` +CREATE TABLE MyTable( + channel varchar, + pv varchar + --xctime bigint + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + groupId='mqTest01', + offsetReset='latest', + topic='mqTest01', + parallelism ='2', + topicIsPattern ='false' + ); -## 6.结果表数据示例： +create table sideTable( + channel varchar, + xccount int, + PRIMARY KEY(channel), + PERIOD FOR SYSTEM_TIME + )WITH( + type='mysql', + url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', + userName='dtstack', + password='abc123', + tableName='sidetest', + cache = 'LRU', + cacheTTLMs='10000', + parallelism ='1' + + ); + + +CREATE TABLE MyResult( + channel varchar, + pv varchar + )WITH( + type='kafka', + bootstrapServers='172.16.8.107:9092', + topic='mqTest02', + parallelism ='2', + updateMode='upsert', + sinkdatatype = 'csv', + fieldDelimiter='*' + + + + ); + + +insert +into + MyResult + select + a.channel as channel, + a.pv as pv + from + MyTable a ``` -[root@node002 bin]# ./kafka-console-consumer.sh --bootstrap-server 172.16.101.224:9092 --topic tiezhu_test_out2 -{"id":122,"name":"tiezhu122","address":"hangzhou122","home":"ganzhou122","school":" ecjtu122"} -{"id":123,"name":"tiezhu123","address":"hangzhou123","home":"ganzhou123","school":" ecjtu123"} -``` \ No newline at end of file diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 60bfc7fc1..88fda90c9 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,6 +1,7 @@ ### 1 插件列表 #### 1.1 源表插件 * [kafka 源表插件](plugin/kafkaSource.md) +* [kafka 结果表插件](plugin/kafkaSink.md) #### 1.2 结果表插件 * [elasticsearch 结果表插件](plugin/elasticsearchSink.md) From 28269884b84b55cc70e5187de366af70739168ed Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 17 Jul 2020 17:43:14 +0800 Subject: [PATCH 275/378] fix --- docs/plugin/hbaseSide.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/plugin/hbaseSide.md b/docs/plugin/hbaseSide.md index 29dc60bf9..550dfbae0 100644 --- a/docs/plugin/hbaseSide.md +++ b/docs/plugin/hbaseSide.md @@ -43,12 +43,13 @@ | tableName | hbase 的表名称|是|| | cache | 维表缓存策略(NONE/LRU)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| -|kerberosAuthEnable | 是否开启kerberos认证|否|false| -|regionserverPrincipal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| -|clientKeytabFile|client的keytab 文件|否| -|clientPrincipal|client的principal|否|| -|zookeeperSaslClient | zookeeper.sasl.client值|否|true| -|securityKrb5Conf | java.security.krb5.conf值|否|| +|hbase.security.auth.enable | 是否开启kerberos认证|否|false| +|hbase.security.authentication|认证类型|否|kerberos| +|hbase.kerberos.regionserver.principal | regionserver的principal，这个值从hbase-site.xml的hbase.regionserver.kerberos.principal属性中获取|否|| +|hbase.keytab|client的keytab 文件|否| +|hbase.principal|client的principal|否|| +|hbase.sasl.clientconfig | hbase.sasl.clientconfig值|否|Client,和jaas文件中的域对应| +|java.security.krb5.conf | java.security.krb5.conf值|否|| 另外开启Kerberos认证还需要在VM参数中配置krb5, -Djava.security.krb5.conf=/Users/xuchao/Documents/flinkSql/kerberos/krb5.conf 同时在addShipfile参数中添加keytab文件的路径，参数具体细节请看[命令参数说明](../config.md) -------------- @@ -226,11 +227,13 @@ CREATE TABLE sideTable( cacheTTLMs ='60000', asyncTimeoutNum ='0', parallelism ='1', - kerberosAuthEnable='true', - regionserverPrincipal='hbase/_HOST@DTSTACK.COM', - clientKeytabFile='test.keytab', - clientPrincipal='test@DTSTACK.COM', - securityKrb5Conf='krb5.conf', + hbase.security.auth.enable='true', + hbase.security.authentication='kerberos', + hbase.sasl.clientconfig='Client', + hbase.kerberos.regionserver.principal='hbase/_HOST@DTSTACK.COM', + hbase.keytab='yijing.keytab', + hbase.principal='yijing@DTSTACK.COM', + java.security.krb5.conf='krb5.conf' ); insert into From 37969bd663b748905d9f4b1c3e4043ed08a7ae2f Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 17 Jul 2020 19:52:18 +0800 Subject: [PATCH 276/378] add error msg --- .../dtstack/flink/sql/util/DataTypeUtils.java | 82 +++++-------------- 1 file changed, 20 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 88ae95ec8..b02ee5504 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.util; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -21,26 +22,21 @@ **/ public class DataTypeUtils { - private final static Pattern COMPOUND_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); + private final static Pattern COMPOSITE_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); private final static String ARRAY = "ARRAY"; private final static String ROW = "ROW"; private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; - private final static Pattern ATOMIC_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*,"); - private final static Pattern COMPLEX_TYPE_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+?>)\\s*,"); - private final static Pattern ATOMIC_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(\\w+)\\s*$"); - private final static Pattern COMPLEX_TAIL_PATTERN = Pattern.compile("^\\s*(\\w+)\\s+(.+?<.+>)\\s*$"); - private DataTypeUtils() {} - public static TypeInformation convertToCollectionType(String string) { - Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + public static TypeInformation convertToCompositeType(String string) { + Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); // TODO 现在只支持ARRAY类型后续可以加入 MAP等类型 if (matcher.find() && ARRAY.equals(matcher.group(1))) { return convertToArray(string); } else { - throw new RuntimeException(""); + throw new RuntimeException("type " + string + "is not support!"); } } @@ -50,7 +46,7 @@ public static TypeInformation convertToCollectionType(String string) { * @return */ public static TypeInformation convertToArray(String arrayTypeString) { - Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(arrayTypeString); + Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(arrayTypeString); if (matcher.find() && ARRAY.equals(matcher.group(1))) { String elementTypeString = matcher.group(2); TypeInformation elementType; @@ -61,7 +57,7 @@ public static TypeInformation convertToArray(String arrayTypeString) { } return Types.OBJECT_ARRAY(elementType); } else { - throw new RuntimeException("convert to array type error!"); + throw new RuntimeException(arrayTypeString + "convert to array type error!"); } } @@ -71,19 +67,17 @@ public static TypeInformation convertToArray(String arrayTypeString) { * @param string */ public static RowTypeInfo convertToRow(String string) { - Matcher matcher = COMPOUND_TYPE_PATTERN.matcher(string); + Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); if (matcher.find() && - ROW.equals( - matcher.group(1).toUpperCase() - ) + ROW.equals(matcher.group(1).toUpperCase()) ) { String elementTypeStr = matcher.group(2); Iterable typeInfo = splitCompositeTypeFields(elementTypeStr); Tuple2 tuple = genFieldInfo(typeInfo); return new RowTypeInfo(tuple.f0, tuple.f1); } else { - throw new RuntimeException(""); + throw new RuntimeException(string + "convert to row type error!"); } } @@ -95,28 +89,26 @@ private static Iterable splitCompositeTypeFields(String string) { } private static Tuple2 genFieldInfo(Iterable typeInfo) { - int fieldsSize = Iterators.size(typeInfo.iterator()); ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); + for (String type : typeInfo) { Iterable fieldInfo = Splitter .on(TYPE_DELIMITER) .trimResults() .omitEmptyStrings() .split(type); + ArrayList array = Lists.newArrayList(fieldInfo.iterator()); - if (array.size() == 2) { - TypeInformation fieldType = convertToAtomicType(array.get(1)); - types.add(fieldType); - fieldNames.add(array.get(0)); - } else { - throw new RuntimeException(); - } + Preconditions.checkState(array.size() == 2, "field info must be name with type"); + TypeInformation fieldType = convertToAtomicType(array.get(1)); + types.add(fieldType); + fieldNames.add(array.get(0)); } - TypeInformation[] typesArray = types.toArray(new TypeInformation[types.size()]); - String[] fieldNamesArray = fieldNames.toArray(new String[fieldNames.size()]); - return Tuple2.of(typesArray, fieldNamesArray); + TypeInformation[] typeArray = types.toArray(new TypeInformation[types.size()]); + String[] fieldNameArray = fieldNames.toArray(new String[fieldNames.size()]); + return Tuple2.of(typeArray, fieldNameArray); } /** @@ -156,42 +148,8 @@ public static TypeInformation convertToAtomicType(String string) { case "TIMESTAMP": return Types.SQL_TIMESTAMP(); default: - throw new RuntimeException("type " + string + "not supported"); + throw new RuntimeException("type " + string + "not supported, please refer to the flink doc!"); } } - /** - * 目前这个方法未使用，设置当初是想字段声明走统一的词法分析器(分词器)。 - * @param fieldStmts - * @return - */ - public static ArrayList fieldStmtLexer(String fieldStmts) { - - String stmtStream = fieldStmts; - ArrayList tokens = new ArrayList<>(); - while (Strings.isNullOrEmpty(stmtStream)) { - Matcher atomicTypeMatcher = ATOMIC_TYPE_PATTERN.matcher(stmtStream); - Matcher complexTypeMatcher = COMPLEX_TYPE_PATTERN.matcher(stmtStream); - Matcher atomicTypeTailMatcher = ATOMIC_TAIL_PATTERN.matcher(stmtStream); - Matcher complexTypeTailMatcher = COMPLEX_TAIL_PATTERN.matcher(stmtStream); - - String fieldStmt; - - if (atomicTypeMatcher.find()) { - fieldStmt = atomicTypeMatcher.group(0); - } else if (complexTypeMatcher.find()) { - fieldStmt = complexTypeMatcher.group(0); - } else if (atomicTypeTailMatcher.find()) { - fieldStmt = atomicTypeTailMatcher.group(0); - } else if (complexTypeTailMatcher.find()) { - fieldStmt = complexTypeTailMatcher.group(0); - } else { - throw new RuntimeException("field declaration statement error" + fieldStmts); - } - - tokens.add(fieldStmt); - stmtStream = stmtStream.substring(fieldStmt.length() + 1); - } - return tokens; - } } From 1ed54a0d93a744d11b6b77e7c3264bac6be44dcf Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 20 Jul 2020 15:03:29 +0800 Subject: [PATCH 277/378] Optimize code structure --- .../dtstack/flink/sql/util/DataTypeUtils.java | 130 +++++++++++------- .../flink/sql/util/DataTypeUtilsTest.java | 29 ++-- 2 files changed, 97 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index b02ee5504..041696787 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -30,14 +30,20 @@ public class DataTypeUtils { private DataTypeUtils() {} - public static TypeInformation convertToCompositeType(String string) { - Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); - // TODO 现在只支持ARRAY类型后续可以加入 MAP等类型 - if (matcher.find() && ARRAY.equals(matcher.group(1))) { - return convertToArray(string); - } else { - throw new RuntimeException("type " + string + "is not support!"); - } + /** + * 现在只支持ARRAY类型后续可以加入 MAP等类型 + * @param compositeTypeString + * @return + */ + public static TypeInformation convertToCompositeType(String compositeTypeString) { + Matcher matcher = matchCompositeType(compositeTypeString); + final String errorMsg = "type " + compositeTypeString + "is not support!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); + + return convertToArray(compositeTypeString); } /** @@ -46,64 +52,56 @@ public static TypeInformation convertToCompositeType(String string) { * @return */ public static TypeInformation convertToArray(String arrayTypeString) { - Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(arrayTypeString); - if (matcher.find() && ARRAY.equals(matcher.group(1))) { - String elementTypeString = matcher.group(2); - TypeInformation elementType; - if (elementTypeString.toUpperCase().startsWith(ROW)) { - elementType = convertToRow(elementTypeString); - } else { - elementType = convertToAtomicType(elementTypeString); - } - return Types.OBJECT_ARRAY(elementType); + Matcher matcher = matchCompositeType(arrayTypeString); + final String errorMsg = arrayTypeString + "convert to array type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); + + String elementTypeString = matcher.group(2); + TypeInformation elementType; + String normalizedElementType = normalizeType(elementTypeString); + if (normalizedElementType.startsWith(ROW)) { + elementType = convertToRow(elementTypeString); } else { - throw new RuntimeException(arrayTypeString + "convert to array type error!"); + elementType = convertToAtomicType(elementTypeString); } + return Types.OBJECT_ARRAY(elementType); } /** * 目前ROW里只支持基本类型 - * @param string + * @param rowTypeString */ - public static RowTypeInfo convertToRow(String string) { - Matcher matcher = COMPOSITE_TYPE_PATTERN.matcher(string); - - if (matcher.find() && - ROW.equals(matcher.group(1).toUpperCase()) - ) { - String elementTypeStr = matcher.group(2); - Iterable typeInfo = splitCompositeTypeFields(elementTypeStr); - Tuple2 tuple = genFieldInfo(typeInfo); - return new RowTypeInfo(tuple.f0, tuple.f1); - } else { - throw new RuntimeException(string + "convert to row type error!"); - } + public static RowTypeInfo convertToRow(String rowTypeString) { + Matcher matcher = matchCompositeType(rowTypeString); + final String errorMsg = rowTypeString + "convert to row type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(ROW.equals(normalizedType), errorMsg); + + String elementTypeStr = matcher.group(2); + Iterable fieldInfos = splitCompositeTypeField(elementTypeStr); + Tuple2 info = genFieldInfo(fieldInfos); + return new RowTypeInfo(info.f0, info.f1); } - private static Iterable splitCompositeTypeFields(String string) { - return Splitter - .on(FIELD_DELIMITER) - .trimResults() - .split(string); - } - private static Tuple2 genFieldInfo(Iterable typeInfo) { + + private static Tuple2 genFieldInfo(Iterable fieldInfos) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); - for (String type : typeInfo) { - Iterable fieldInfo = Splitter - .on(TYPE_DELIMITER) - .trimResults() - .omitEmptyStrings() - .split(type); - - ArrayList array = Lists.newArrayList(fieldInfo.iterator()); - Preconditions.checkState(array.size() == 2, "field info must be name with type"); - TypeInformation fieldType = convertToAtomicType(array.get(1)); + for (String fieldInfo : fieldInfos) { + Iterable splitedInfo = splitTypeInfo(fieldInfo); + ArrayList info = Lists.newArrayList(splitedInfo.iterator()); + Preconditions.checkState(info.size() == 2, "field info must be name with type"); + TypeInformation fieldType = convertToAtomicType(info.get(1)); types.add(fieldType); - fieldNames.add(array.get(0)); + fieldNames.add(info.get(0)); } TypeInformation[] typeArray = types.toArray(new TypeInformation[types.size()]); @@ -117,7 +115,7 @@ private static Tuple2 genFieldInfo(Iterable * @return */ public static TypeInformation convertToAtomicType(String string) { - switch (string.toUpperCase()) { + switch (normalizeType(string)) { case "VARCHAR": case "STRING": return Types.STRING(); @@ -152,4 +150,32 @@ public static TypeInformation convertToAtomicType(String string) { } } + private static Iterable splitTypeInfo(String string) { + return Splitter + .on(TYPE_DELIMITER) + .trimResults() + .omitEmptyStrings() + .split(string); + } + + private static Iterable splitCompositeTypeField(String string) { + return Splitter + .on(FIELD_DELIMITER) + .trimResults() + .split(string); + } + + private static String replaceBlank(String s) { + return s.replaceAll("\\s", " ").trim(); + } + + private static Matcher matchCompositeType(String s) { + return COMPOSITE_TYPE_PATTERN.matcher( + replaceBlank(s) + ); + } + + private static String normalizeType(String s) { + return s.toUpperCase().trim(); + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java index 4d5627348..a8fbf88fc 100644 --- a/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/util/DataTypeUtilsTest.java @@ -15,26 +15,35 @@ public class DataTypeUtilsTest { @Test public void convertToArray() { + String[] normalFieldNames = new String[] { "id", "name" }; + TypeInformation[] normalTypes = new TypeInformation[] {Types.INT(), Types.STRING()}; + RowTypeInfo rowTypeInfo = new RowTypeInfo(normalTypes, normalFieldNames); + TypeInformation normalAtomicType = Types.OBJECT_ARRAY(Types.STRING()); + TypeInformation normalCompositeType = Types.OBJECT_ARRAY(rowTypeInfo); + + String atomicStrWithBlank = " ARRAY< STRING\n > "; + String compositeTypeStrWithBlank = " ARRAY< ROW< id INT, name STRING > > "; + + TypeInformation atomicArrayTypeWithBlank = DataTypeUtils.convertToArray(atomicStrWithBlank); + TypeInformation compositeTypeArrayTypeWithBlank = DataTypeUtils.convertToArray(compositeTypeStrWithBlank); + + Assert.assertTrue(normalAtomicType.equals(atomicArrayTypeWithBlank)); + Assert.assertTrue(normalCompositeType.equals(compositeTypeArrayTypeWithBlank)); + + String atomicStr = "ARRAY"; String compositeTypeStr = "ARRAY>"; TypeInformation atomicArrayType = DataTypeUtils.convertToArray(atomicStr); TypeInformation compositeTypeArrayType = DataTypeUtils.convertToArray(compositeTypeStr); - - String[] normalFieldNames = new String[] {"id", "name"}; - TypeInformation[] normalTypes = new TypeInformation[] {Types.INT(), Types.STRING()}; - RowTypeInfo rowTypeInfo = new RowTypeInfo(normalTypes, normalFieldNames); - TypeInformation normalAtomicType = Types.OBJECT_ARRAY(Types.STRING()); - TypeInformation normalCompositeType = Types.OBJECT_ARRAY(rowTypeInfo); - Assert.assertTrue(normalAtomicType.equals(atomicArrayType)); Assert.assertTrue(normalCompositeType.equals(compositeTypeArrayType)); } @Test public void convertToRow() { - String string = "ROW"; + String string = " ROW < id INT, name STRING > "; RowTypeInfo rowType = DataTypeUtils.convertToRow(string); String[] fieldNames = rowType.getFieldNames(); @@ -48,7 +57,7 @@ public void convertToRow() { @Test public void convertToAtomicType() { - TypeInformation type = DataTypeUtils.convertToAtomicType("STRING"); - Assert.assertTrue(type == Types.STRING()); + TypeInformation type = DataTypeUtils.convertToAtomicType(" TIMESTAMP "); + Assert.assertTrue(type == Types.SQL_TIMESTAMP()); } } \ No newline at end of file From f6ce914ee08b787429d2607389f876dc7058f40e Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 20 Jul 2020 16:43:28 +0800 Subject: [PATCH 278/378] optimize structure --- .../flink/sql/table/AbstractTableParser.java | 18 ++++++++---------- .../dtstack/flink/sql/util/DataTypeUtils.java | 15 +++++++-------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index beed69fcb..5eaee1c6f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -103,16 +103,15 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ } // 处理复合类型，例如 ARRAY> + // 把ARRAY类型的长串字符压入Buffer Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); - if ( - !tailMatcher.matches() && - (headMatcher.matches() || - !buffer.isEmpty()) - ) { + boolean isNotTail = !tailMatcher.matches(); + boolean isToNeedPush = headMatcher.matches() || !buffer.isEmpty(); + + if (isNotTail && isToNeedPush) { writeBuffer(buffer, fieldRow); } else { - String[] fieldInfoArr; if (tailMatcher.matches()) { buffer.add(fieldRow); @@ -122,9 +121,8 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ fieldInfoArr = fieldRow.split("\\s+"); } - if (fieldInfoArr.length < 2) { - throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); - } + String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); + Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); if (isMatcherKey) { @@ -149,7 +147,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ fieldClass = dbTypeConvertToJavaType(fieldType); } - tableInfo.addPhysicalMappings(fieldInfoArr[0],fieldInfoArr[0]); + tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); tableInfo.addField(fieldName); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 041696787..ff368b975 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -84,24 +84,23 @@ public static RowTypeInfo convertToRow(String rowTypeString) { Preconditions.checkState(ROW.equals(normalizedType), errorMsg); String elementTypeStr = matcher.group(2); - Iterable fieldInfos = splitCompositeTypeField(elementTypeStr); - Tuple2 info = genFieldInfo(fieldInfos); + Iterable fieldInfoStrs = splitCompositeTypeField(elementTypeStr); + Tuple2 info = genFieldInfo(fieldInfoStrs); return new RowTypeInfo(info.f0, info.f1); } - - - private static Tuple2 genFieldInfo(Iterable fieldInfos) { + private static Tuple2 genFieldInfo(Iterable fieldInfoStrs) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); - for (String fieldInfo : fieldInfos) { - Iterable splitedInfo = splitTypeInfo(fieldInfo); + for (String fieldStr : fieldInfoStrs) { + Iterable splitedInfo = splitTypeInfo(fieldStr); ArrayList info = Lists.newArrayList(splitedInfo.iterator()); Preconditions.checkState(info.size() == 2, "field info must be name with type"); + + fieldNames.add(info.get(0)); TypeInformation fieldType = convertToAtomicType(info.get(1)); types.add(fieldType); - fieldNames.add(info.get(0)); } TypeInformation[] typeArray = types.toArray(new TypeInformation[types.size()]); From 4e2a1df6a17d5c3f6c6b40fecc6654c043ebda90 Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 20 Jul 2020 16:44:32 +0800 Subject: [PATCH 279/378] add space --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 5eaee1c6f..2bebb75a4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -95,10 +95,10 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ ArrayList buffer = new ArrayList<>(); - for(String fieldRow : fieldRows){ + for (String fieldRow : fieldRows) { fieldRow = fieldRow.trim(); - if(StringUtils.isBlank(fieldRow)){ + if (StringUtils.isBlank(fieldRow)) { throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } From faafbd0db0a9383b99c20967ba279478a7367e4c Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 21 Jul 2020 14:20:48 +0800 Subject: [PATCH 280/378] add license and doc, update split part --- .../flink/sql/table/AbstractTableParser.java | 84 +++++++------------ .../dtstack/flink/sql/util/DataTypeUtils.java | 21 ++++- .../dtstack/flink/sql/util/DtStringUtil.java | 6 ++ docs/plugin/kafkaSource.md | 80 +++++++++++++++++- 4 files changed, 132 insertions(+), 59 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 2bebb75a4..8173dafe9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -16,8 +16,6 @@ * limitations under the License. */ - - package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.util.ClassUtil; @@ -27,7 +25,6 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -52,9 +49,6 @@ public abstract class AbstractTableParser { private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern compositeTypeHeadPattern = Pattern.compile(".+<.+<.+"); - private static Pattern compositeTypeTailPattern = Pattern.compile(".*>\\s*>.*"); - private Map patternMap = Maps.newHashMap(); private Map handlerMap = Maps.newHashMap(); @@ -89,12 +83,10 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ return false; } - public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ + public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); - ArrayList buffer = new ArrayList<>(); - for (String fieldRow : fieldRows) { fieldRow = fieldRow.trim(); @@ -102,63 +94,45 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){ throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - // 处理复合类型，例如 ARRAY> - // 把ARRAY类型的长串字符压入Buffer - Matcher headMatcher = compositeTypeHeadPattern.matcher(fieldRow); - Matcher tailMatcher = compositeTypeTailPattern.matcher(fieldRow); - boolean isNotTail = !tailMatcher.matches(); - boolean isToNeedPush = headMatcher.matches() || !buffer.isEmpty(); + String[] fieldInfoArr = fieldRow.split("\\s+"); - if (isNotTail && isToNeedPush) { - writeBuffer(buffer, fieldRow); - } else { - String[] fieldInfoArr; - if (tailMatcher.matches()) { - buffer.add(fieldRow); - fieldRow = String.join("", buffer); - fieldInfoArr = readBuffer(buffer); - } else { - fieldInfoArr = fieldRow.split("\\s+"); - } + String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); + Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); - String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); - Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); + boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); + if (isMatcherKey) { + continue; + } - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); - if (isMatcherKey) { - continue; - } + //Compatible situation may arise in space in the fieldName + String[] filedNameArr = new String[fieldInfoArr.length - 1]; + System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); + String fieldName = String.join(" ", filedNameArr); + String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); - //Compatible situation may arise in space in the fieldName - String[] filedNameArr = new String[fieldInfoArr.length - 1]; - System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); - String fieldName = String.join(" ", filedNameArr); - String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); - - Class fieldClass = null; - AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; - - Matcher matcher = charTypePattern.matcher(fieldType); - if (matcher.find()) { - fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); - fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); - } else { - fieldClass = dbTypeConvertToJavaType(fieldType); - } + Class fieldClass = null; + AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; - tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); - tableInfo.addField(fieldName); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); + Matcher matcher = charTypePattern.matcher(fieldType); + if (matcher.find()) { + fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); + fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); + fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + } else { + fieldClass = dbTypeConvertToJavaType(fieldType); } + + tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); + tableInfo.addField(fieldName); + tableInfo.addFieldClass(fieldClass); + tableInfo.addFieldType(fieldType); + tableInfo.addFieldExtraInfo(fieldExtraInfo); } tableInfo.finish(); } - public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){ + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { String primaryFields = matcher.group(1).trim(); String[] splitArry = primaryFields.split(","); List primaryKes = Lists.newArrayList(splitArry); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index ff368b975..72a8b4b66 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -1,8 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.util; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -11,7 +27,6 @@ import org.apache.flink.table.api.Types; import java.util.ArrayList; -import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index b4200d733..37fbf9c2b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -89,6 +89,12 @@ public static List splitIgnoreQuota(String str, char delimiter){ } else if (c == ')' && !inSingleQuotes && !inQuotes) { bracketLeftNum--; b.append(c); + } else if (c == '<' && !inSingleQuotes && !inQuotes) { + bracketLeftNum++; + b.append(c); + } else if (c == '>' && !inSingleQuotes && !inQuotes) { + bracketLeftNum--; + b.append(c); } else { b.append(c); } diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 9e628f287..67afc45a1 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -130,7 +130,85 @@ CREATE TABLE MyTable( ); ``` -数组类型字段解析示例 +#### 数组类型字段解析示例 + +声明数据类型为Array可以使用CROSS JOIN UNNEST语句对Array类型进行展开。（新模式，推荐） + +json样例 +``` +{ + "id": 2, + "some_users": [ + "foo", + "bar" + ] +} +``` + +SQL样例 +``` +CREATE TABLE ods ( + id INT, + some_users ARRAY> +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + user_no INT, + user_info VARCHAR +) WITH ( + type ='console', +); + +INSERT INTO dwd + SELECT id, user_info + FROM ods_foo + CROSS JOIN UNNEST(ods_foo.some_users) AS A (user_info); +``` + +json样例 +``` +{ + "id": 4, + "some_users": [ + { + "user_no": 12, + "user_info": "foo" + + }, + { + "user_no": 14, + "user_info": "bar" + } + ] + +``` + +SQL样例 +``` +CREATE TABLE ods ( + id INT, + some_users ARRAY> +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + user_no INT, + user_info VARCHAR +) WITH ( + type ='console', +); + +INSERT INTO dwd + SELECT id, user_no, user_info + FROM ods_foo + CROSS JOIN UNNEST(ods_foo.some_users) AS A (user_no, user_info); +``` +##### 旧模式 json: {"name":"tom", "obj":{"channel": "root"}, "user": [{"pv": 4}, {"pv": 10}], "xctime":1572932485} ``` From 87f272b1e78e3f2b6dd39d600403794f743e7513 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 21 Jul 2020 14:24:08 +0800 Subject: [PATCH 281/378] fix kafka source doc --- docs/plugin/kafkaSource.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin/kafkaSource.md b/docs/plugin/kafkaSource.md index 67afc45a1..cb7823900 100644 --- a/docs/plugin/kafkaSource.md +++ b/docs/plugin/kafkaSource.md @@ -149,7 +149,7 @@ SQL样例 ``` CREATE TABLE ods ( id INT, - some_users ARRAY> + some_users ARRAY ) WITH ( ... ); @@ -183,7 +183,7 @@ json样例 "user_info": "bar" } ] - +} ``` SQL样例 From 4112f78cc38d632bebc839780e560a7f832d7327 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 21 Jul 2020 14:41:00 +0800 Subject: [PATCH 282/378] remove useless method --- .../flink/sql/table/AbstractTableParser.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 8173dafe9..371de6d5d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -25,7 +25,6 @@ import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -172,23 +171,4 @@ protected void addParserHandler(String parserName, Pattern pattern, ITableFieldD handlerMap.put(parserName, handler); } - private void writeBuffer(List buffer, String fieldRow) { - StringBuilder builder = new StringBuilder(); - builder.append(fieldRow); - builder.append(","); - buffer.add(builder.toString()); - } - - private String[] readBuffer(List buffer) { - String fieldRow = String.join("", buffer); - buffer.clear(); - String[] tmp = fieldRow.split("\\s+"); - String[] type = Arrays.copyOfRange(tmp, 1, tmp.length); - String[] fieldInfoArr = new String[] { - tmp[0], - String.join(" ", type) - }; - return fieldInfoArr; - } - } From 0ff459b1686a683bac5ef7bffcb5eb694dcc7123 Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 5 Aug 2020 14:04:04 +0800 Subject: [PATCH 283/378] fix parse error when much space before AS --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 2 +- core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 371de6d5d..82b653297 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -44,7 +44,7 @@ public abstract class AbstractTableParser { private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 5fa96aca9..ec88bdbe9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -37,7 +37,7 @@ public class ClassUtil { public static Class stringConvertClass(String str) { // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型 - String lowerStr = str.toLowerCase(); + String lowerStr = str.toLowerCase().trim(); if (lowerStr.startsWith("array")) { return Array.newInstance(Integer.class, 0).getClass(); } From 6b28712b934fc69593ef3191b969cb3bef3dee10 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 6 Aug 2020 10:31:15 +0800 Subject: [PATCH 284/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=20=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=86=85=E5=AE=B9=E5=92=8C=E6=A0=BC=E5=BC=8F=E4=B8=8D?= =?UTF-8?q?=E5=AF=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index fbb0aa291..d4d1505f8 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -141,7 +141,7 @@ public void accept(Map values) { if (MapUtils.isNotEmpty(values)) { try { Row row = fillData(input.row(), values); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.MultiLine, values)); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(input, resultFuture, e); From fd6fa5f7b22df1a65090bd82f88f2e836ad7b9a3 Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 6 Aug 2020 15:16:43 +0800 Subject: [PATCH 285/378] =?UTF-8?q?fix=20join=E9=87=8D=E5=A4=8Dtablename?= =?UTF-8?q?=E4=B8=8D=E6=B3=A8=E5=86=8C=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/side/JoinInfo.java | 11 ++++---- .../flink/sql/side/JoinNodeDealer.java | 25 +++++++++++-------- .../dtstack/flink/sql/side/SideSQLParser.java | 23 +++++++++-------- .../dtstack/flink/sql/util/TableUtils.java | 17 +++++++++++++ 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 997ed417e..f37b82882 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -69,6 +69,8 @@ public class JoinInfo implements Serializable { private String scope = ""; + private String newTableName = null; + /** * 左表需要查询的字段信息和output的时候对应的列名称 */ @@ -96,13 +98,12 @@ public String getNonSideTable(){ } public String getNewTableName(){ - //兼容左边表是as 的情况 - String leftStr = leftTableName; - leftStr = Strings.isNullOrEmpty(leftStr) ? leftTableAlias : leftStr; - String newName = leftStr + "_" + rightTableName; - return TableUtils.buildTableNameWithScope(newName, scope); + return this.newTableName; } + public void setNewTableName(String newTableName){ + this.newTableName = newTableName; + } public String getNewTableAlias(){ String newName = leftTableAlias + "_" + rightTableAlias; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index e963fae65..57104bd04 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -91,7 +91,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Set> joinFieldSet, Map tableRef, Map fieldRef, - String scope) { + String scope, + Set joinTableNames) { SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); @@ -108,13 +109,13 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, if (leftNode.getKind() == JOIN) { //处理连续join dealNestJoin(joinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, - parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope, joinTableNames); leftNode = joinNode.getLeft(); } if (leftNode.getKind() == AS) { AliasInfo aliasInfo = (AliasInfo) sideSQLParser.parseSql(leftNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, scope); + parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); leftTbName = aliasInfo.getName(); leftTbAlias = aliasInfo.getAlias(); } else if(leftNode.getKind() == IDENTIFIER){ @@ -126,7 +127,7 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, Preconditions.checkState(!leftIsSide, "side-table must be at the right of join operator"); Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, scope); + parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); rightTableName = rightTableNameAndAlias.f0; rightTableAlias = rightTableNameAndAlias.f1; @@ -150,7 +151,8 @@ public JoinInfo dealJoinNode(SqlJoin joinNode, tableInfo.setJoinType(joinType); tableInfo.setCondition(joinNode.getCondition()); tableInfo.setScope(scope); - + tableInfo.setNewTableName(TableUtils.buildTableNameWithScope(leftTbName, leftTbAlias, rightTableName, scope, joinTableNames)); + joinTableNames.add(tableInfo.getNewTableName()); TableUtils.replaceJoinFieldRefTableName(joinNode.getCondition(), fieldRef); //extract 需要查询的字段信息 @@ -262,20 +264,21 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, Set> joinFieldSet, Map tableRef, Map fieldRef, - String scope){ + String scope, + Set joinTableNames){ SqlJoin leftJoinNode = (SqlJoin) joinNode.getLeft(); SqlNode parentRightJoinNode = joinNode.getRight(); SqlNode rightNode = leftJoinNode.getRight(); Tuple2 rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, scope); + parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); Tuple2 parentRightJoinInfo = parseRightNode(parentRightJoinNode, sideTableSet, - queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); boolean parentRightIsSide = checkIsSideTable(parentRightJoinInfo.f0, sideTableSet); JoinInfo joinInfo = dealJoinNode(leftJoinNode, sideTableSet, queueInfo, parentWhere, parentSelectList, - parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); + parentGroupByList, joinFieldSet, tableRef, fieldRef, scope, joinTableNames); String rightTableName = rightTableNameAndAlias.f0; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); @@ -670,12 +673,12 @@ private void extractSelectField(SqlNode selectNode, private Tuple2 parseRightNode(SqlNode sqlNode, Set sideTableSet, Queue queueInfo, SqlNode parentWhere, SqlNodeList selectList, SqlNodeList parentGroupByList, - String scope) { + String scope, Set joinTableNames) { Tuple2 tabName = new Tuple2<>("", ""); if(sqlNode.getKind() == IDENTIFIER){ tabName.f0 = sqlNode.toString(); }else{ - AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList, scope); + AliasInfo aliasInfo = (AliasInfo)sideSQLParser.parseSql(sqlNode, sideTableSet, queueInfo, parentWhere, selectList, parentGroupByList, scope, joinTableNames); tabName.f0 = aliasInfo.getName(); tabName.f1 = aliasInfo.getAlias(); } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java index ba568144d..d94073b69 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSQLParser.java @@ -71,7 +71,7 @@ public Queue getExeQueue(String exeSql, Set sideTableSet, String FlinkPlannerImpl flinkPlanner = FlinkPlanner.getFlinkPlanner(); SqlNode sqlNode = flinkPlanner.parse(exeSql); - parseSql(sqlNode, sideTableSet, queueInfo, null, null, null, scope); + parseSql(sqlNode, sideTableSet, queueInfo, null, null, null, scope, Sets.newHashSet()); queueInfo.offer(sqlNode); return queueInfo; } @@ -92,7 +92,8 @@ public Object parseSql(SqlNode sqlNode, SqlNode parentWhere, SqlNodeList parentSelectList, SqlNodeList parentGroupByList, - String scope){ + String scope, + Set joinTableNames){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ case WITH: { @@ -100,15 +101,15 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList sqlNodeList = sqlWith.withList; for (SqlNode withAsTable : sqlNodeList) { SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(sqlWithItem.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); queueInfo.add(sqlWithItem); } - parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(sqlWith.body, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); break; } case INSERT: SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); - return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + return parseSql(sqlSource, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); case SELECT: SqlNode sqlFrom = ((SqlSelect)sqlNode).getFrom(); SqlNode sqlWhere = ((SqlSelect)sqlNode).getWhere(); @@ -116,7 +117,7 @@ public Object parseSql(SqlNode sqlNode, SqlNodeList groupByList = ((SqlSelect) sqlNode).getGroup(); if(sqlFrom.getKind() != IDENTIFIER){ - Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList, scope); + Object result = parseSql(sqlFrom, sideTableSet, queueInfo, sqlWhere, selectList, groupByList, scope, joinTableNames); if(result instanceof JoinInfo){ return TableUtils.dealSelectResultWithJoinInfo((JoinInfo) result, (SqlSelect) sqlNode, queueInfo); }else if(result instanceof AliasInfo){ @@ -138,7 +139,7 @@ public Object parseSql(SqlNode sqlNode, Map tableRef = Maps.newHashMap(); Map fieldRef = Maps.newHashMap(); return joinNodeDealer.dealJoinNode((SqlJoin) sqlNode, sideTableSet, queueInfo, - parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef, scope); + parentWhere, parentSelectList, parentGroupByList, joinFieldSet, tableRef, fieldRef, scope, joinTableNames); case AS: SqlNode info = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode alias = ((SqlBasicCall) sqlNode).getOperands()[1]; @@ -147,7 +148,7 @@ public Object parseSql(SqlNode sqlNode, if(info.getKind() == IDENTIFIER){ infoStr = info.toString(); } else { - infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope).toString(); + infoStr = parseSql(info, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames).toString(); } AliasInfo aliasInfo = new AliasInfo(); @@ -160,12 +161,12 @@ public Object parseSql(SqlNode sqlNode, SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); - parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(unionLeft, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); + parseSql(unionRight, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); break; case ORDER_BY: SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope); + parseSql(sqlOrderBy.query, sideTableSet, queueInfo, parentWhere, parentSelectList, parentGroupByList, scope, joinTableNames); case LITERAL: return LITERAL.toString(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index c6befb86d..94878355b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -41,6 +41,7 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.table.api.Table; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.stream.Collector; import static org.apache.calcite.sql.SqlKind.*; import static org.apache.calcite.sql.SqlKind.CASE; @@ -710,4 +712,19 @@ public static String buildTableNameWithScope(String tableName, String scope){ return tableName + "_" + scope; } + public static String buildTableNameWithScope(String leftTableName, String leftTableAlias, String rightTableName, String scope, Set existTableNames){ + //兼容左边表是as 的情况 + String leftStr = Strings.isNullOrEmpty(leftTableName) ? leftTableAlias : leftTableName; + String newName = leftStr + "_" + rightTableName; + String newTableName = TableUtils.buildTableNameWithScope(newName, scope); + if (CollectionUtils.isEmpty(existTableNames)) { + return newName; + } + if (!existTableNames.contains(newName)) { + return newName; + } + return newName + "_" + System.currentTimeMillis(); + + } + } From ff5ba0720bea30d8564a7065a2f092605444242e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 11 Aug 2020 20:20:15 +0800 Subject: [PATCH 286/378] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=20DataUtil?= =?UTF-8?q?=EF=BC=8CEs6Sink=20=E4=B8=AD=20java.util.Date=20=E5=92=8C=20jav?= =?UTF-8?q?a.sql.Date=20=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtnest/DtNestRowDeserializationSchema.java | 6 +----- .../dtstack/flink/sql/table/AbstractTableInfo.java | 8 +++----- .../java/com/dtstack/flink/sql/util/DateUtil.java | 13 +++++++++++++ .../sql/sink/elasticsearch/CustomerSinkFunc.java | 10 +++++++++- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 3a6ca4829..13cc261ad 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -106,11 +106,7 @@ private void parseTree(JsonNode jsonNode, String prefix) { String nodeKey = getNodeKey(prefix, next); nodeAndJsonNodeMapping.put(nodeKey, child); - if (child.isArray()) { - parseTree(child, nodeKey); - } else { - parseTree(child, nodeKey); - } + parseTree(child, nodeKey); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 3b514dd48..40b7a7e82 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -16,8 +16,6 @@ * limitations under the License. */ - - package com.dtstack.flink.sql.table; import com.google.common.collect.Lists; @@ -178,9 +176,9 @@ public void setPhysicalFields(Map physicalFields) { } public void finish(){ - this.fields = fieldList.toArray(new String[fieldList.size()]); - this.fieldClasses = fieldClassList.toArray(new Class[fieldClassList.size()]); - this.fieldTypes = fieldTypeList.toArray(new String[fieldTypeList.size()]); + this.fields = fieldList.toArray(new String[0]); + this.fieldClasses = fieldClassList.toArray(new Class[0]); + this.fieldTypes = fieldTypeList.toArray(new String[0]); } /** diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index a420c9d82..b05de0f16 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -108,9 +108,22 @@ public static Date localDateTimetoDate(LocalDateTime localDateTime){ } public static LocalDateTime dateToLocalDateTime(Date date){ + date = transformSqlDateToUtilDate(date); return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); } + /** + * 将java.sql.Date 转化为 java.util.Date + * @param date 不知道是java.sql.Date 还是 java.util.Date + * @return 最后返回 java.util.Date + */ + public static Date transformSqlDateToUtilDate(Date date) { + if (date instanceof java.sql.Date) { + date = new Date(date.getTime()); + } + return date; + } + /** * * diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index ec1732d0e..d594ddd93 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.util.DateUtil; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.metrics.Counter; @@ -31,6 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -42,7 +44,9 @@ public class CustomerSinkFunc implements ElasticsearchSinkFunction { private final Logger logger = LoggerFactory.getLogger(CustomerSinkFunc.class); - /** 用作ID的属性值连接符号 */ + /** + * 用作ID的属性值连接符号 + */ private static final String ID_VALUE_SPLIT = "_"; private String index; @@ -107,6 +111,10 @@ private IndexRequest createIndexRequest(Row element) { Map dataMap = Es6Util.rowToJsonMap(element, fieldNames, fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); for (int i = 0; i < length; i++) { + if (element.getField(i) instanceof Date) { + dataMap.put(fieldNames.get(i), DateUtil.transformSqlDateToUtilDate((Date) element.getField(i))); + continue; + } dataMap.put(fieldNames.get(i), element.getField(i)); } From be084d7177fce96294a3914ebe29310e6a604a04 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 11 Aug 2020 20:41:29 +0800 Subject: [PATCH 287/378] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E5=A4=A7=E5=B0=8F=E5=86=99=E4=B8=8D=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E5=8A=A0=E8=BD=BD=E6=8F=92=E4=BB=B6=E4=B8=8D?= =?UTF-8?q?=E6=AD=A3=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 6f2fbd9ea..f58a0c9a0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -321,7 +321,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { - String sideOperator = ECacheType.ALL.name().equals(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; + String sideOperator = ECacheType.ALL.name().equalsIgnoreCase(((AbstractSideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; sideTableMap.put(tableInfo.getName(), (AbstractSideTableInfo) tableInfo); URL sideTablePathUrl = PluginUtil.buildSidePathByLoadMode(tableInfo.getType(), sideOperator, AbstractSideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); From d417e519815c9528bfbf8c2caed7f62d0b1f676b Mon Sep 17 00:00:00 2001 From: dapeng Date: Thu, 13 Aug 2020 11:42:20 +0800 Subject: [PATCH 288/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BB=B4=E8=A1=A8?= =?UTF-8?q?=E8=A1=A8=E6=98=8E=E6=B2=A1=E6=9C=89=E5=B8=A6=E6=9C=89scope?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exec/ExecuteProcessHelper.java | 2 +- .../dtstack/flink/sql/util/TableUtils.java | 25 ++++--------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 6f2fbd9ea..24595f902 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -229,7 +229,7 @@ private static void sqlTranslation(String localSqlPluginPath, } if (isSide) { //sql-dimensional table contains the dimension table of execution - sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null, null); + sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache, queryConfig, null, String.valueOf(scope)); } else { LOG.info("----------exec sql without dimension join-----------"); LOG.info("----------real sql exec is--------------------------\n{}", result.getExecSql()); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 94878355b..2aeb06cf8 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -26,8 +26,6 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.HashBiMap; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.typesafe.config.ConfigException; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDataTypeSpec; @@ -49,7 +47,6 @@ import java.util.Map; import java.util.Queue; import java.util.Set; -import java.util.stream.Collector; import static org.apache.calcite.sql.SqlKind.*; import static org.apache.calcite.sql.SqlKind.CASE; @@ -191,25 +188,13 @@ public static void extractSelectFieldToFieldInfo(SqlNode fieldNode, String fromN } } - public static String buildInternalTableName(String left, char split, String right) { - StringBuilder sb = new StringBuilder(); - return sb.append(left).append(split).append(right).toString(); - } - public static SqlBasicCall buildAsNodeByJoinInfo(JoinInfo joinInfo, SqlNode sqlNode0, String tableAlias) { SqlOperator operator = new SqlAsOperator(); SqlParserPos sqlParserPos = new SqlParserPos(0, 0); String newTableName = joinInfo.getNewTableName(); - String lefTbAlias = joinInfo.getLeftTableAlias(); - if(Strings.isNullOrEmpty(lefTbAlias)){ - Set fromTableSet = Sets.newHashSet(); - TableUtils.getFromTableInfo(joinInfo.getLeftNode(), fromTableSet); - lefTbAlias = StringUtils.join(fromTableSet, "_"); - } - - String newTableAlias = !StringUtils.isEmpty(tableAlias) ? tableAlias : buildInternalTableName(lefTbAlias, SPLIT, joinInfo.getRightTableAlias()); + String newTableAlias = !StringUtils.isEmpty(tableAlias) ? tableAlias : joinInfo.getNewTableAlias(); if (null == sqlNode0) { sqlNode0 = new SqlIdentifier(newTableName, null, sqlParserPos); @@ -716,15 +701,15 @@ public static String buildTableNameWithScope(String leftTableName, String leftTa //兼容左边表是as 的情况 String leftStr = Strings.isNullOrEmpty(leftTableName) ? leftTableAlias : leftTableName; String newName = leftStr + "_" + rightTableName; - String newTableName = TableUtils.buildTableNameWithScope(newName, scope); if (CollectionUtils.isEmpty(existTableNames)) { - return newName; + return TableUtils.buildTableNameWithScope(newName, scope); } + if (!existTableNames.contains(newName)) { - return newName; + return TableUtils.buildTableNameWithScope(newName, scope); } - return newName + "_" + System.currentTimeMillis(); + return TableUtils.buildTableNameWithScope(newName, scope) + "_" + System.currentTimeMillis(); } } From 18740de1f166fe0d74f2377473b3ca3f8bd6626d Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 14 Aug 2020 14:43:11 +0800 Subject: [PATCH 289/378] fix <> parse error --- .../flink/sql/table/AbstractTableParser.java | 2 +- .../dtstack/flink/sql/util/DtStringUtil.java | 52 +++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index 371de6d5d..3b9bf7922 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -84,7 +84,7 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { - List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); + List fieldRows = DtStringUtil.splitField(fieldsInfo); for (String fieldRow : fieldRows) { fieldRow = fieldRow.trim(); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 37fbf9c2b..3e2ebb82a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -46,14 +46,61 @@ public class DtStringUtil { private static ObjectMapper objectMapper = new ObjectMapper(); - /** * Split the specified string delimiter --- ignored quotes delimiter * @param str * @param delimiter * @return */ - public static List splitIgnoreQuota(String str, char delimiter){ + public static List splitIgnoreQuota(String str, char delimiter) { + List tokensList = new ArrayList<>(); + boolean inQuotes = false; + boolean inSingleQuotes = false; + int bracketLeftNum = 0; + StringBuilder b = new StringBuilder(); + char[] chars = str.toCharArray(); + int idx = 0; + for (char c : chars) { + char flag = 0; + if (idx > 0) { + flag = chars[idx - 1]; + } + if (c == delimiter) { + if (inQuotes) { + b.append(c); + } else if (inSingleQuotes) { + b.append(c); + } else if (bracketLeftNum > 0) { + b.append(c); + } else { + tokensList.add(b.toString()); + b = new StringBuilder(); + } + } else if (c == '\"' && '\\' != flag && !inSingleQuotes) { + inQuotes = !inQuotes; + b.append(c); + } else if (c == '\'' && '\\' != flag && !inQuotes) { + inSingleQuotes = !inSingleQuotes; + b.append(c); + } else if (c == '(' && !inSingleQuotes && !inQuotes) { + bracketLeftNum++; + b.append(c); + } else if (c == ')' && !inSingleQuotes && !inQuotes) { + bracketLeftNum--; + b.append(c); + } else { + b.append(c); + } + idx++; + } + + tokensList.add(b.toString()); + + return tokensList; + } + + public static List splitField(String str) { + final char delimiter = ','; List tokensList = new ArrayList<>(); boolean inQuotes = false; boolean inSingleQuotes = false; @@ -106,7 +153,6 @@ public static List splitIgnoreQuota(String str, char delimiter){ return tokensList; } - public static String replaceIgnoreQuota(String str, String oriStr, String replaceStr){ String splitPatternStr = oriStr + "(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)(?=(?:[^']*'[^']*')*[^']*$)"; return str.replaceAll(splitPatternStr, replaceStr); From b64dd57aab965268d053144c9143e7a1dea242f5 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 14 Aug 2020 17:03:18 +0800 Subject: [PATCH 290/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=BA=E5=B0=91?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java index f3b983fb4..86bd7c1a7 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseConfigUtils.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.hbase; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.HadoopKerberosName; +import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; From eb23e15fac177b4f67b81100f0f7955335e42384 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 19 Aug 2020 11:42:00 +0800 Subject: [PATCH 291/378] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8Dpostgre=20?= =?UTF-8?q?=E5=81=9A=E7=BB=B4=E8=A1=A8=E6=97=B6schema=E4=B8=8D=E7=94=9F?= =?UTF-8?q?=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/postgresql/PostgresqlAllSideInfo.java | 7 +++++++ .../flink/sql/side/postgresql/PostgresqlAsyncReqRow.java | 6 +----- .../flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java | 7 +++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java index a4a52eabc..fa07d078b 100644 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -38,4 +40,9 @@ public class PostgresqlAllSideInfo extends RdbAllSideInfo { public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } } diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java index 1ded64371..3470b6b8f 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java @@ -19,10 +19,9 @@ package com.dtstack.flink.sql.side.postgresql; -import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; import io.vertx.core.Vertx; @@ -35,9 +34,6 @@ import org.slf4j.LoggerFactory; import java.util.List; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * Date: 2019-08-11 diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java index f19a488bc..ddb365fe0 100644 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java @@ -22,6 +22,8 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; +import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -39,4 +41,9 @@ public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } + + @Override + public String getTableName(RdbSideTableInfo rdbSideTableInfo) { + return DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName()); + } } From 49e5a257475096bccb72107131338fa6191d8c5d Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 26 Aug 2020 11:35:56 +0800 Subject: [PATCH 292/378] =?UTF-8?q?fix=20=E5=86=85=E5=AD=98=E6=BA=A2?= =?UTF-8?q?=E5=87=BA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch6/Elasticsearch6AsyncReqRow.java | 11 +++++++++-- .../flink/sql/side/elasticsearch6/util/Es6Util.java | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java index fa6d507bb..8be1c1f3a 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-async-side/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/Elasticsearch6AsyncReqRow.java @@ -70,6 +70,7 @@ public class Elasticsearch6AsyncReqRow extends BaseAsyncReqRow implements Serial private transient RestHighLevelClient rhlClient; private SearchRequest searchRequest; private List sqlJoinCompareOperate = Lists.newArrayList(); + private static final Integer MAX_ROW_NUM = 50000; public Elasticsearch6AsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new Elasticsearch6AsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -92,7 +93,7 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul String key = buildCacheKey(inputParams); BoolQueryBuilder boolQueryBuilder = Es6Util.setPredicateclause(sideInfo); boolQueryBuilder = setInputParams(inputParams, boolQueryBuilder); - SearchSourceBuilder searchSourceBuilder = initConfiguration(); + SearchSourceBuilder searchSourceBuilder = initConfiguration(inputParams); searchSourceBuilder.query(boolQueryBuilder); searchRequest.source(searchSourceBuilder); @@ -116,6 +117,11 @@ public void onResponse(SearchResponse searchResponse) { if (searchHits.length < getFetchSize()) { break; } + //protect memory + if (rowList.size() >= MAX_ROW_NUM) { + LOG.warn("row size beyond limit"); + break; + } if (tableInfo == null && tmpRhlClient == null) { // create new connection to fetch data tableInfo = (Elasticsearch6SideTableInfo) sideInfo.getSideTableInfo(); @@ -225,10 +231,11 @@ public void close() throws Exception { } - private SearchSourceBuilder initConfiguration() { + private SearchSourceBuilder initConfiguration(Map inputParams) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(getFetchSize()); searchSourceBuilder.sort("_id", SortOrder.DESC); + inputParams.keySet().stream().forEach(k -> searchSourceBuilder.sort(k, SortOrder.DESC)); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields().trim(), ","); searchSourceBuilder.fetchSource(sideFieldNames, null); diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java index bc5637515..4a25b43d9 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/Es6Util.java @@ -171,6 +171,9 @@ public static BoolQueryBuilder buildFilterCondition(BoolQueryBuilder boolQueryBu return boolQueryBuilder.must(QueryBuilders.existsQuery(info.getFieldName())); case "=": case "EQUALS": + if(StringUtils.isBlank(info.getCondition())){ + return boolQueryBuilder; + } return boolQueryBuilder.must(QueryBuilders.termQuery(textConvertToKeyword(info.getFieldName(), sideInfo), removeSpaceAndApostrophe(info.getCondition())[0])); case "<>": case "NOT_EQUALS": From 6cc401fefb29eccbcb53c94bdf8cce2be7204daf Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 1 Sep 2020 09:56:43 +0800 Subject: [PATCH 293/378] =?UTF-8?q?=E5=8D=87=E7=BA=A7flink=E7=9A=84?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 374ec3c31..0496c6dd8 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ UTF-8 - 1.8.1 + 1.8.3 From 9e7bd2b4b7b0ed898549dd7185cc62e16d9b1e36 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 1 Sep 2020 16:30:52 +0800 Subject: [PATCH 294/378] fix array type parse error --- .../flink/sql/table/AbstractTableParser.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index b5c463cb6..ade1ebb42 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.tuple.Tuple2; import java.util.List; import java.util.Map; @@ -47,6 +48,8 @@ public abstract class AbstractTableParser { private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(.+)"); + private Map patternMap = Maps.newHashMap(); @@ -93,23 +96,16 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName())); } - String[] fieldInfoArr = fieldRow.split("\\s+"); - - String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow); - Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg); - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); if (isMatcherKey) { continue; } - //Compatible situation may arise in space in the fieldName - String[] filedNameArr = new String[fieldInfoArr.length - 1]; - System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1); - String fieldName = String.join(" ", filedNameArr); - String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim(); + Tuple2 t = extractType(fieldRow, tableInfo.getName()); + String fieldName = t.f0; + String fieldType = t.f1; - Class fieldClass = null; + Class fieldClass; AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null; Matcher matcher = charTypePattern.matcher(fieldType); @@ -121,7 +117,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { fieldClass = dbTypeConvertToJavaType(fieldType); } - tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]); + tableInfo.addPhysicalMappings(fieldName, fieldName); tableInfo.addField(fieldName); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); @@ -131,11 +127,23 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { tableInfo.finish(); } + private Tuple2 extractType(String fieldRow, String tableName) { + Matcher matcher = typePattern.matcher(fieldRow); + if (matcher.matches()) { + String fieldName = matcher.group(1); + String fieldType = matcher.group(2); + return Tuple2.of(fieldName, fieldType); + } else { + String errorMsg = String.format("table [%s] field [%s] format error.", tableName, fieldRow); + throw new RuntimeException(errorMsg); + } + } + public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { String primaryFields = matcher.group(1).trim(); - String[] splitArry = primaryFields.split(","); - List primaryKes = Lists.newArrayList(splitArry); - tableInfo.setPrimaryKeys(primaryKes); + String[] splitArray = primaryFields.split(","); + List primaryKeys = Lists.newArrayList(splitArray); + tableInfo.setPrimaryKeys(primaryKeys); } /** From e7e64e9958a7c05e708ec0191e952dc9ecd6c5fb Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 2 Sep 2020 09:26:14 +0800 Subject: [PATCH 295/378] fix regular of making error msg invalid --- .../java/com/dtstack/flink/sql/table/AbstractTableParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index ade1ebb42..f0898cfd6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -48,7 +48,7 @@ public abstract class AbstractTableParser { private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(.+)"); + private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); private Map patternMap = Maps.newHashMap(); From 533725bf1739667062d61dde97e101972ccff15b Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 3 Sep 2020 16:39:03 +0800 Subject: [PATCH 296/378] hbase sink failed --- .../java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java index 7af51be3d..13329faed 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -110,10 +110,6 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ - throw new RuntimeException(field + " \n" + - "Format defined exceptions"); - } //判断是不是常量==>''包裹的标识 if(field.startsWith("'") && field.endsWith("'")){ From 3b433faca7bee93e333cf695305b7698016af4a2 Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 3 Sep 2020 17:11:11 +0800 Subject: [PATCH 297/378] hbase sink failed doc fixed --- docs/plugin/hbaseSink.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/plugin/hbaseSink.md b/docs/plugin/hbaseSink.md index 5006f11a2..a5ad12067 100644 --- a/docs/plugin/hbaseSink.md +++ b/docs/plugin/hbaseSink.md @@ -7,7 +7,7 @@ CREATE TABLE MyResult( type ='hbase', zookeeperQuorum ='ip:port[,ip:port]', tableName ='tableName', - rowKey ='colName[,colName]', + rowKey ='colName[+colName]', parallelism ='1', zookeeperParent ='/hbase' ) @@ -34,7 +34,7 @@ hbase2.0 |zookeeperQuorum | hbase zk地址,多个直接用逗号隔开|是|| |zookeeperParent | zkParent 路径|是|| |tableName | 关联的hbase表名称|是|| -|rowkey | hbase的rowkey关联的列信息,多个值以逗号隔开|是|| +|rowkey | hbase的rowkey关联的列信息,多个值以'+'连接|是|| |updateMode|APPEND：不回撤数据，只下发增量数据，UPSERT：先删除回撤数据，然后更新|否|APPEND｜ |parallelism | 并行度设置|否|1| |kerberosAuthEnable | 是否开启kerberos认证|否|false| @@ -76,7 +76,7 @@ CREATE TABLE MyResult( tableName ='myresult', partitionedJoin ='false', parallelism ='1', - rowKey='name,channel' + rowKey='name+channel' ); insert @@ -141,7 +141,7 @@ into ## 6.hbase数据 ### 数据内容说明 -hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'-'连接 +hbase的rowkey 构建规则：以描述的rowkey字段值作为key，多个字段以'+'连接 ### 数据内容示例 hbase(main):007:0> scan 'myresult' ROW COLUMN+CELL From 278bcc4e13cea6e207c1ab436a17a258f4d4e2c8 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 8 Sep 2020 15:01:46 +0800 Subject: [PATCH 298/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Doracle=20=E8=81=94?= =?UTF-8?q?=E5=90=88=E4=B8=BB=E9=94=AE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java index 3a320658c..590ecc7a1 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleDialect.java @@ -104,7 +104,7 @@ private String buildConnectionByAllReplace(boolean allReplace, String col) { private String buildConnectionConditions(String[] uniqueKeyFields) { - return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(",")); + return Arrays.stream(uniqueKeyFields).map(col -> "T1." + quoteIdentifier(col) + "=T2." + quoteIdentifier(col)).collect(Collectors.joining(" AND ")); } /** From 840edc74fec5ef94fce45450d27d8121f80962c2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 8 Sep 2020 16:40:05 +0800 Subject: [PATCH 299/378] =?UTF-8?q?=E8=A1=A8=E4=B8=8D=E5=AD=98=E5=9C=A8?= =?UTF-8?q?=E6=97=B6=E4=BB=BB=E5=8A=A1=E7=BB=93=E6=9D=9F=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 7c3ff4b09..9dd06d6a0 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -118,11 +118,18 @@ public void executeUpdate(Connection connection) { } catch (SQLException e1) { throw new RuntimeException(e1); } + + if(e.getMessage().contains("doesn't exist")){ + throw new RuntimeException("table not exist"); + + } if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); LOG.error("", e); } metricOutputFormat.outDirtyRecords.inc(); + + } }); rows.clear(); From 1586a62f0d97695c5af49b6c290375c616b262ff Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 8 Sep 2020 17:12:53 +0800 Subject: [PATCH 300/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java index 9dd06d6a0..b1d601c22 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AppendOnlyWriter.java @@ -120,7 +120,7 @@ public void executeUpdate(Connection connection) { } if(e.getMessage().contains("doesn't exist")){ - throw new RuntimeException("table not exist"); + throw new RuntimeException(e); } if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { From df5fcccb398071cf65c7f361c2d326e575e504ee Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 18 Sep 2020 09:55:21 +0800 Subject: [PATCH 301/378] =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E8=87=B4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index ec88bdbe9..cad1c47cb 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -48,8 +48,10 @@ public static Class stringConvertClass(String str) { return Boolean.class; case "smallint": + return Short.class; case "smallintunsigned": case "tinyint": + return Byte.class; case "tinyintunsigned": case "mediumint": case "mediumintunsigned": From 5d0bd0c840a4976bc8392e8ac000c8d68d823366 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 18 Sep 2020 11:07:59 +0800 Subject: [PATCH 302/378] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AE=9A=E6=97=B6flu?= =?UTF-8?q?sh=20=E5=A4=B1=E8=B4=A5=E6=A0=87=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index 4e047492f..b87c2e042 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -41,6 +41,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -76,6 +77,7 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat tuple2) throws IOException { + if(!flushFlag.get()){ + throw new RuntimeException("connect exception,can not write record"); + } checkConnectionOpen(); try { if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { @@ -187,6 +193,7 @@ private void checkConnectionOpen() { public synchronized void flush() throws Exception { jdbcWriter.executeBatch(connection); batchCount = 0; + flushFlag.set(true); } /** From 3b2f3648240113a895bb45e4c71de4e82045c844 Mon Sep 17 00:00:00 2001 From: dapeng Date: Fri, 18 Sep 2020 16:58:11 +0800 Subject: [PATCH 303/378] =?UTF-8?q?impala=20jdbc=20=E6=94=AF=E6=8C=81=20ch?= =?UTF-8?q?ar=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index ded966c68..b1a31de42 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -204,6 +204,8 @@ public static int[] buildSqlTypes(List fieldTypeArray) { tmpFieldsType[i] = Types.TINYINT; } else if (fieldType.equals(Short.class.getName())) { tmpFieldsType[i] = Types.SMALLINT; + } else if(fieldType.equals(Character.class.getName())){ + tmpFieldsType[i] = Types.CHAR; } else if (fieldType.equals(String.class.getName())) { tmpFieldsType[i] = Types.CHAR; } else if (fieldType.equals(Byte.class.getName())) { From f33d68ff856f02057215cf8cce9ead3c35261ffa Mon Sep 17 00:00:00 2001 From: wuren Date: Sat, 19 Sep 2020 15:55:59 +0800 Subject: [PATCH 304/378] remove add file statment --- .../dtstack/flink/sql/parser/SqlParser.java | 20 ++++++++ .../flink/sql/parser/SqlParserTest.java | 51 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 23f8e4942..badcf8d9c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -28,8 +28,11 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Reason: @@ -51,6 +54,8 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } + private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+"); + /** * flink support sql syntax * CREATE TABLE sls_stream() with (); @@ -74,6 +79,7 @@ public static SqlTree parseSql(String sql) throws Exception { .replace("\t", " ").trim(); List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); + sqlArr = removeAddFileStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ @@ -154,4 +160,18 @@ public static SqlTree parseSql(String sql) throws Exception { return sqlTree; } + + /** + * remove add file with statment etc. add file /etc/krb5.conf; + */ + private static List removeAddFileStmt(List stmts) { + List cleanedStmts = new ArrayList<>(); + for (String stmt : stmts) { + Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt); + if(!matcher.matches()) { + cleanedStmts.add(stmt); + } + } + return cleanedStmts; + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java new file mode 100644 index 000000000..fdbb03a75 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.parser; + +import org.junit.Assert; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.util.ArrayList; +import java.util.List; + +/** + * @program: flink.sql + * @author: wuren + * @create: 2020/09/15 + **/ +public class SqlParserTest { + + @Test + public void testRemoveAddFileStmt() throws Exception { + List rawStmts = new ArrayList<>(); + String sql1 = " add file asdasdasd "; + String sql2 = " aDd fIle With asdasdasd "; + String sql3 = " INSERT INTO dwd_foo SELECT id, name FROM ods_foo"; + String sql4 = " ADD FILE asb "; + rawStmts.add(sql1); + rawStmts.add(sql2); + rawStmts.add(sql3); + rawStmts.add(sql4); + + List stmts = Whitebox.invokeMethod(SqlParser.class, "removeAddFileStmt", rawStmts); + Assert.assertEquals(stmts.get(0), sql3); + } + +} \ No newline at end of file From ad867fefd4b099cc06740dd52e96426965af9c7f Mon Sep 17 00:00:00 2001 From: dapeng Date: Mon, 21 Sep 2020 11:22:41 +0800 Subject: [PATCH 305/378] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E5=80=BC=E4=B8=8D=E5=AF=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d4d1505f8..88332c21b 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -141,7 +141,7 @@ public void accept(Map values) { if (MapUtils.isNotEmpty(values)) { try { Row row = fillData(input.row(), values); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, row)); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, values)); resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); } catch (Exception e) { dealFillDataError(input, resultFuture, e); From 4864dc79223c507d832b6f562dcd942993508659 Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 21 Sep 2020 15:00:54 +0800 Subject: [PATCH 306/378] fix hbase async with kafka kerberos login error --- .gitignore | 2 ++ .../sql/side/hbase/HbaseAsyncReqRow.java | 34 +++++++++++++++---- .../side/hbase/utils/HbaseConfigUtils.java | 13 ++----- .../{resource => resources}/log4j.properties | 0 4 files changed, 32 insertions(+), 17 deletions(-) rename launcher/src/main/{resource => resources}/log4j.properties (100%) diff --git a/.gitignore b/.gitignore index d7842e1ed..c8f97e79e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ lib/ bin/nohup.out .DS_Store bin/sideSql.txt +krb5.conf +*.keytab \ No newline at end of file diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 0250decae..8d082a51a 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -31,10 +31,14 @@ import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; import com.stumbleupon.async.Deferred; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.security.DynamicConfiguration; +import org.apache.flink.runtime.security.KerberosUtils; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -45,7 +49,12 @@ import org.slf4j.LoggerFactory; import sun.security.krb5.KrbException; +import javax.security.auth.login.AppConfigurationEntry; +import java.io.File; +import java.lang.reflect.Field; +import java.security.PrivilegedExceptionAction; import java.sql.Timestamp; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -100,7 +109,7 @@ public void open(Configuration parameters) throws Exception { ExecutorService executorService =new ThreadPoolExecutor(DEFAULT_POOL_SIZE, DEFAULT_POOL_SIZE, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-aysnc")); + new LinkedBlockingQueue<>(), new DTThreadFactory("hbase-async")); Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); @@ -111,12 +120,15 @@ public void open(Configuration parameters) throws Exception { }); if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { - String jaasStr = HbaseConfigUtils.buildJaasStr(hbaseConfig); - String jaasFilePath = HbaseConfigUtils.creatJassFile(jaasStr); - System.setProperty(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); - config.overrideConfig(HbaseConfigUtils.KEY_JAVA_SECURITY_AUTH_LOGIN_CONF, jaasFilePath); + String principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_PRINCIPAL); + String keytab = System.getProperty("user.dir") + File.separator + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); + LOG.info("Kerberos login with keytab: {} and principal: {}", keytab, principal); + String name = "HBaseClient"; + config.overrideConfig("hbase.sasl.clientconfig", name); + appendJaasConf(name, keytab, principal); refreshConfig(); } + hBaseClient = new HBaseClient(config, executorService); try { @@ -144,11 +156,21 @@ public void open(Configuration parameters) throws Exception { } } + private void appendJaasConf(String name, String keytab, String principal) { + javax.security.auth.login.Configuration priorConfig = javax.security.auth.login.Configuration.getConfiguration(); + // construct a dynamic JAAS configuration + DynamicConfiguration currentConfig = new DynamicConfiguration(priorConfig); + // wire up the configured JAAS login contexts to use the krb5 entries + AppConfigurationEntry krb5Entry = KerberosUtils.keytabEntry(keytab, principal); + currentConfig.addAppConfigurationEntry(name, krb5Entry); + javax.security.auth.login.Configuration.setConfiguration(currentConfig); + } + private void refreshConfig() throws KrbException { sun.security.krb5.Config.refresh(); KerberosName.resetDefaultRealm(); //reload java.security.auth.login.config - javax.security.auth.login.Configuration.setConfiguration(null); +// javax.security.auth.login.Configuration.setConfiguration(null); } @Override diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index c6399e27a..c588b178b 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -33,7 +33,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.UUID; /** * @@ -60,7 +59,7 @@ public class HbaseConfigUtils { private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; public static final String KEY_KEY_TAB = "hbase.keytab"; - private static final String KEY_PRINCIPAL = "hbase.principal"; + public static final String KEY_PRINCIPAL = "hbase.principal"; public final static String KEY_HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; public final static String KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM = "hbase.zookeeper.znode.parent"; @@ -115,9 +114,6 @@ public static boolean asyncOpenKerberos(Map hbaseConfigMap) { return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); } - - - public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { for (String key : KEYS_KERBEROS_REQUIRED) { if (StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))) { @@ -176,16 +172,13 @@ public static String creatJassFile(String configStr) throws IOException { return temp.getAbsolutePath(); } - public static String buildJaasStr(Map kerberosConfig) { + public static String buildJaasStr(Map kerberosConfig,String principal,String keyTab) { for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); } } - String keyTab = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_KEY_TAB); - String principal = MapUtils.getString(kerberosConfig, KEY_PRINCIPAL); - StringBuilder jaasSB = new StringBuilder("Client {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" + " useKeyTab=true\n" + @@ -196,8 +189,6 @@ public static String buildJaasStr(Map kerberosConfig) { return jaasSB.toString(); } - - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); diff --git a/launcher/src/main/resource/log4j.properties b/launcher/src/main/resources/log4j.properties similarity index 100% rename from launcher/src/main/resource/log4j.properties rename to launcher/src/main/resources/log4j.properties From bf2e3095a6b1f77e46af04c66287a3f0b08dd283 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 21 Sep 2020 15:51:37 +0800 Subject: [PATCH 307/378] [fix] fix hbase data dislocate --- .../dtstack/flink/sql/table/AbstractTableInfo.java | 2 +- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java index 40b7a7e82..f8fa9c3ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableInfo.java @@ -49,7 +49,7 @@ public abstract class AbstractTableInfo implements Serializable { private final List fieldList = Lists.newArrayList(); /**key:别名, value: realField */ - private Map physicalFields = Maps.newHashMap(); + private Map physicalFields = Maps.newLinkedHashMap(); private final List fieldTypeList = Lists.newArrayList(); diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 904c75a1b..6c31bd8dd 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -27,7 +27,11 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.AuthUtil; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Put; @@ -39,9 +43,9 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedAction; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; /** * @author: jingzhen@dtstack.com @@ -327,8 +331,8 @@ public HbaseOutputFormat finish() { String[] qualifiers = new String[format.columnNames.length]; if (format.columnNameFamily != null) { - Set keySet = format.columnNameFamily.keySet(); - String[] columns = keySet.toArray(new String[keySet.size()]); + List keyList = new LinkedList<>(format.columnNameFamily.keySet()); + String[] columns = keyList.toArray(new String[0]); for (int i = 0; i < columns.length; ++i) { String col = columns[i]; String[] part = col.split(":"); From e4082d2e25768bc6f678cb7a9a48d1f9533d709c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 21 Sep 2020 16:27:46 +0800 Subject: [PATCH 308/378] [fix] fix hbase rowkey format defined --- .../java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java index 7af51be3d..d0911dbcb 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/RowKeyBuilder.java @@ -110,7 +110,7 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ + if(field.length() <= 0){ throw new RuntimeException(field + " \n" + "Format defined exceptions"); } From 22dd152dfabeac6e07ef2a8a1def2fb06804b93f Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 21 Sep 2020 21:24:03 +0800 Subject: [PATCH 309/378] add error msg and remove useless method --- .../dtstack/flink/sql/util/DtFileUtils.java | 36 +++++++++++++++ .../sql/side/hbase/HbaseAsyncReqRow.java | 12 ++++- .../sql/side/hbase/HbaseAsyncSideInfo.java | 18 ++++++++ .../side/hbase/utils/HbaseConfigUtils.java | 44 +++---------------- 4 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java new file mode 100644 index 000000000..eb1c974df --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtFileUtils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.apache.flink.util.Preconditions; + +import java.io.File; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/21 + **/ +public class DtFileUtils { + public static void checkExists(String path) { + File file = new File(path); + String errorMsg = "%s file is not exist!"; + Preconditions.checkState(file.exists(), errorMsg, path); + } +} diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 8d082a51a..f799f3439 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -30,6 +30,9 @@ import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.side.hbase.utils.HbaseConfigUtils; +import com.dtstack.flink.sql.util.DtFileUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.stumbleupon.async.Deferred; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -114,14 +117,21 @@ public void open(Configuration parameters) throws Exception { Config config = new Config(); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_QUORUM, hbaseSideTableInfo.getHost()); config.overrideConfig(HbaseConfigUtils.KEY_HBASE_ZOOKEEPER_ZNODE_QUORUM, hbaseSideTableInfo.getParent()); - HbaseConfigUtils.loadKrb5Conf(hbaseConfig); hbaseConfig.entrySet().forEach(entity -> { config.overrideConfig(entity.getKey(), (String) entity.getValue()); }); if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + HbaseConfigUtils.loadKrb5Conf(hbaseConfig); + String principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_PRINCIPAL); + Preconditions.checkState(!Strings.isNullOrEmpty(principal), "%s must be set!", HbaseConfigUtils.KEY_PRINCIPAL); + String regionserver_principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + Preconditions.checkState(!Strings.isNullOrEmpty(regionserver_principal), "%s must be set!", HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); String keytab = System.getProperty("user.dir") + File.separator + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); + DtFileUtils.checkExists(keytab); LOG.info("Kerberos login with keytab: {} and principal: {}", keytab, principal); String name = "HBaseClient"; config.overrideConfig("hbase.sasl.clientconfig", name); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java index 135aec004..6601458d4 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.side.FieldInfo; diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index c588b178b..bdb4ffdf9 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.hbase.utils; +import com.dtstack.flink.sql.util.DtFileUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -26,9 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -57,7 +56,7 @@ public class HbaseConfigUtils { // async side kerberos private final static String KEY_HBASE_SECURITY_AUTH_ENABLE = "hbase.security.auth.enable"; private final static String KEY_HBASE_SASL_CLIENTCONFIG = "hbase.sasl.clientconfig"; - private final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; + public final static String KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL = "hbase.kerberos.regionserver.principal"; public static final String KEY_KEY_TAB = "hbase.keytab"; public static final String KEY_PRINCIPAL = "hbase.principal"; @@ -100,13 +99,6 @@ public static Configuration getConfig(Map hbaseConfigMap) { return hConfiguration; } - public static boolean openKerberos(Map hbaseConfigMap) { - if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION)) { - return false; - } - return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); - } - public static boolean asyncOpenKerberos(Map hbaseConfigMap) { if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { return false; @@ -156,39 +148,13 @@ public static String getKeytab(Map hbaseConfigMap) { public static void loadKrb5Conf(Map kerberosConfig) { String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + DtFileUtils.checkExists(krb5FilePath); if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { - System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath);; + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); + LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } } - public static String creatJassFile(String configStr) throws IOException { - String fileName = System.getProperty("user.dir"); - File krbConf = new File(fileName); - File temp = File.createTempFile("JAAS", ".conf", krbConf); - temp.deleteOnExit(); - BufferedWriter out = new BufferedWriter(new FileWriter(temp, false)); - out.write(configStr + "\n"); - out.close(); - return temp.getAbsolutePath(); - } - - public static String buildJaasStr(Map kerberosConfig,String principal,String keyTab) { - for (String key : ASYNC_KEYS_KERBEROS_REQUIRED) { - if (StringUtils.isEmpty(MapUtils.getString(kerberosConfig, key))) { - throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key)); - } - } - - StringBuilder jaasSB = new StringBuilder("Client {\n" + - " com.sun.security.auth.module.Krb5LoginModule required\n" + - " useKeyTab=true\n" + - " useTicketCache=false\n"); - jaasSB.append(" keyTab=\"").append(keyTab).append("\"").append("\n"); - jaasSB.append(" principal=\"").append(principal).append("\"").append(";\n"); - jaasSB.append("};"); - return jaasSB.toString(); - } - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); From 215460328796d80ce2f07d62d7951e9b2bb411ed Mon Sep 17 00:00:00 2001 From: wuren Date: Mon, 21 Sep 2020 22:00:07 +0800 Subject: [PATCH 310/378] update krb5 error msg --- .../sql/side/hbase/utils/HbaseConfigUtils.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index bdb4ffdf9..cdd48b640 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -19,6 +19,8 @@ package com.dtstack.flink.sql.side.hbase.utils; import com.dtstack.flink.sql.util.DtFileUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -146,13 +148,13 @@ public static String getKeytab(Map hbaseConfigMap) { throw new IllegalArgumentException(""); } - public static void loadKrb5Conf(Map kerberosConfig) { - String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(kerberosConfig, KEY_JAVA_SECURITY_KRB5_CONF); + public static void loadKrb5Conf(Map config) { + String krb5conf = MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); + Preconditions.checkState(!Strings.isNullOrEmpty(krb5conf), "%s must be set!", KEY_JAVA_SECURITY_KRB5_CONF); + String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); DtFileUtils.checkExists(krb5FilePath); - if (!org.apache.commons.lang.StringUtils.isEmpty(krb5FilePath)) { - System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); - LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); - } + System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); + LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { From 28fa8c76fc18d5b99f5f2fd869941c1f7943aacd Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 22 Sep 2020 10:32:21 +0800 Subject: [PATCH 311/378] optmize code --- .../sql/side/hbase/HbaseAsyncReqRow.java | 16 +++++----- .../side/hbase/utils/HbaseConfigUtils.java | 31 ++++++++++++++++--- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index f799f3439..c36c4affb 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -121,17 +121,17 @@ public void open(Configuration parameters) throws Exception { config.overrideConfig(entity.getKey(), (String) entity.getValue()); }); - if (HbaseConfigUtils.asyncOpenKerberos(hbaseConfig)) { + if (HbaseConfigUtils.isEnableKerberos(hbaseConfig)) { HbaseConfigUtils.loadKrb5Conf(hbaseConfig); - String principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_PRINCIPAL); - Preconditions.checkState(!Strings.isNullOrEmpty(principal), "%s must be set!", HbaseConfigUtils.KEY_PRINCIPAL); - String regionserver_principal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); - Preconditions.checkState(!Strings.isNullOrEmpty(regionserver_principal), "%s must be set!", HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + HbaseConfigUtils.checkOpt(principal, HbaseConfigUtils.KEY_PRINCIPAL); + String regionserverPrincipal = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + HbaseConfigUtils.checkOpt(regionserverPrincipal, HbaseConfigUtils.KEY_HBASE_KERBEROS_REGIONSERVER_PRINCIPAL); + String keytab = MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); + HbaseConfigUtils.checkOpt(keytab, HbaseConfigUtils.KEY_KEY_TAB); + String keytabPath = System.getProperty("user.dir") + File.separator + keytab; + DtFileUtils.checkExists(keytabPath); - MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); - String keytab = System.getProperty("user.dir") + File.separator + MapUtils.getString(hbaseConfig, HbaseConfigUtils.KEY_KEY_TAB); - DtFileUtils.checkExists(keytab); LOG.info("Kerberos login with keytab: {} and principal: {}", keytab, principal); String name = "HBaseClient"; config.overrideConfig("hbase.sasl.clientconfig", name); diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java index cdd48b640..cad4fccc9 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/utils/HbaseConfigUtils.java @@ -101,11 +101,27 @@ public static Configuration getConfig(Map hbaseConfigMap) { return hConfiguration; } - public static boolean asyncOpenKerberos(Map hbaseConfigMap) { - if (!MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE)) { - return false; + public static boolean isEnableKerberos(Map hbaseConfigMap) { + boolean hasAuthorization = AUTHENTICATION_TYPE.equalsIgnoreCase( + MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHORIZATION) + ); + boolean hasAuthentication = AUTHENTICATION_TYPE.equalsIgnoreCase( + MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION) + ); + boolean hasAuthEnable = MapUtils.getBooleanValue(hbaseConfigMap, KEY_HBASE_SECURITY_AUTH_ENABLE); + + if(hasAuthentication || hasAuthorization || hasAuthEnable) { + LOG.info("Enable kerberos for hbase."); + setKerberosConf(hbaseConfigMap); + return true; } - return AUTHENTICATION_TYPE.equalsIgnoreCase(MapUtils.getString(hbaseConfigMap, KEY_HBASE_SECURITY_AUTHENTICATION)); + return false; + } + + private static void setKerberosConf(Map hbaseConfigMap) { + hbaseConfigMap.put(KEY_HBASE_SECURITY_AUTHORIZATION, AUTHENTICATION_TYPE); + hbaseConfigMap.put(KEY_HBASE_SECURITY_AUTHENTICATION, AUTHENTICATION_TYPE); + hbaseConfigMap.put(KEY_HBASE_SECURITY_AUTH_ENABLE, true); } public static Configuration getHadoopConfiguration(Map hbaseConfigMap) { @@ -150,13 +166,18 @@ public static String getKeytab(Map hbaseConfigMap) { public static void loadKrb5Conf(Map config) { String krb5conf = MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); - Preconditions.checkState(!Strings.isNullOrEmpty(krb5conf), "%s must be set!", KEY_JAVA_SECURITY_KRB5_CONF); + checkOpt(krb5conf, KEY_JAVA_SECURITY_KRB5_CONF); String krb5FilePath = System.getProperty("user.dir") + File.separator + MapUtils.getString(config, KEY_JAVA_SECURITY_KRB5_CONF); DtFileUtils.checkExists(krb5FilePath); System.setProperty(KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); LOG.info("{} is set to {}", KEY_JAVA_SECURITY_KRB5_CONF, krb5FilePath); } + // TODO 日后改造可以下沉到Core模块 + public static void checkOpt(String opt, String key) { + Preconditions.checkState(!Strings.isNullOrEmpty(opt), "%s must be set!", key); + } + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String principal, String keytab) throws IOException { if (conf == null) { throw new IllegalArgumentException("kerberos conf can not be null"); From bf17ce0568d0136c6c9a333a575730e086c32baa Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 22 Sep 2020 20:23:30 +0800 Subject: [PATCH 312/378] [fix] fix hbase rowkey format defined --- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 6acfcb760..f80bf9f22 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -101,7 +101,7 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ + if(field.length() <= 0){ throw new RuntimeException(field + " \n" + "Format defined exceptions"); } From e1f0662777282e5bf6138205f90c2ab459617173 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 22 Sep 2020 22:13:17 +0800 Subject: [PATCH 313/378] =?UTF-8?q?[fix]=20fix=20=E7=BB=B4=E8=A1=A8join?= =?UTF-8?q?=E5=87=BA=E7=8E=B0Table=20not=20Found?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/JoinNodeDealer.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 57104bd04..6afc12462 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -489,7 +489,7 @@ private Set extractSelectFieldFromJoinCondition(Set extractSelectFieldFromJoinCondition(Set fromTableNameSet, String checkTableName, Map mappingTableName) { + for (int i = 0; i < mappingTableName.size() + 1; i++) { + if (fromTableNameSet.contains(checkTableName)) { + return true; + } + + checkTableName = mappingTableName.get(checkTableName); + if (checkTableName == null) { + return false; + } + } + return true; + } + private Set extractFieldFromGroupByList(SqlNodeList parentGroupByList, Set fromTableNameSet, Map tableRef){ From 79e0842e796cd5aa0f454c3aa83564dd48aedd2b Mon Sep 17 00:00:00 2001 From: chuixue Date: Thu, 24 Sep 2020 09:31:49 +0800 Subject: [PATCH 314/378] [30589][kafka-sink]remove kafka sink retract data --- .../java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 7234216a7..1d0c4d0e2 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -100,6 +100,7 @@ protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] public void emitDataStream(DataStream> dataStream) { DataStream mapDataStream = dataStream .map((Tuple2 record) -> new CRow(record.f1, record.f0)) + .filter(x -> x.change()) .returns(getRowTypeInfo()) .setParallelism(parallelism); From c90af4d8de11d00b7d6667169f3079aa8a24dc1c Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 24 Sep 2020 17:26:42 +0800 Subject: [PATCH 315/378] [fix] sink add parallelism --- .../flink/sql/sink/cassandra/CassandraSink.java | 9 ++++++++- .../sql/sink/elasticsearch/ElasticsearchSink.java | 10 ++++++++-- .../elasticsearch/table/ElasticsearchSinkParser.java | 3 +++ .../sql/sink/elasticsearch/ElasticsearchSink.java | 10 ++++++++-- .../elasticsearch/table/ElasticsearchSinkParser.java | 3 +++ .../com/dtstack/flink/sql/sink/hbase/HbaseSink.java | 4 ++-- .../flink/sql/sink/kafka/AbstractKafkaSink.java | 2 +- .../flink/sql/sink/kafka/table/KafkaSinkParser.java | 2 +- .../com/dtstack/flink/sql/sink/kudu/KuduSink.java | 5 ++++- .../com/dtstack/flink/sql/sink/mongo/MongoSink.java | 11 ++++++++++- .../com/dtstack/flink/sql/sink/redis/RedisSink.java | 11 ++++++++++- .../flink/sql/sink/redis/table/RedisSinkParser.java | 5 +++-- 12 files changed, 61 insertions(+), 14 deletions(-) diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java index 26152a7d3..a78ef97f7 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraSink.java @@ -34,6 +34,8 @@ import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; +import java.util.Objects; + /** * Reason: * Date: 2018/11/22 @@ -57,6 +59,8 @@ public class CassandraSink implements RetractStreamTableSink, IStreamSinkGe protected Integer readTimeoutMillis; protected Integer connectTimeoutMillis; protected Integer poolTimeoutMillis; + protected Integer parallelism = 1; + protected String registerTableName; public CassandraSink() { // TO DO NOTHING @@ -77,6 +81,9 @@ public CassandraSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.readTimeoutMillis = cassandraTableInfo.getReadTimeoutMillis(); this.connectTimeoutMillis = cassandraTableInfo.getConnectTimeoutMillis(); this.poolTimeoutMillis = cassandraTableInfo.getPoolTimeoutMillis(); + this.parallelism = Objects.isNull(cassandraTableInfo.getParallelism()) ? + parallelism : cassandraTableInfo.getParallelism(); + this.registerTableName = cassandraTableInfo.getTableName(); return this; } @@ -100,7 +107,7 @@ public void emitDataStream(DataStream> dataStream) { CassandraOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction).setParallelism(parallelism).name(registerTableName); } @Override diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index 5ca81c5ed..6958804fa 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * table output elastic5plugin @@ -75,7 +76,9 @@ public class ElasticsearchSink implements RetractStreamTableSink, IStreamSi private TypeInformation[] fieldTypes; - private int parallelism = -1; + private int parallelism = 1; + + private String registerTableName; private ElasticsearchTableInfo esTableInfo; @@ -149,7 +152,7 @@ private RichSinkFunction createEsSinkFunction(){ @Override public void emitDataStream(DataStream> dataStream) { RichSinkFunction richSinkFunction = createEsSinkFunction(); - DataStreamSink streamSink = dataStream.addSink(richSinkFunction); + DataStreamSink streamSink = dataStream.addSink(richSinkFunction).name(registerTableName); if(parallelism > 0){ streamSink.setParallelism(parallelism); } @@ -176,6 +179,9 @@ public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) String id = elasticsearchTableInfo.getId(); String[] idField = StringUtils.split(id, ","); idIndexList = new ArrayList<>(); + parallelism = Objects.isNull(elasticsearchTableInfo.getParallelism()) ? + parallelism : elasticsearchTableInfo.getParallelism(); + registerTableName = elasticsearchTableInfo.getName(); for(int i = 0; i < idField.length; ++i) { idIndexList.add(Integer.valueOf(idField[i])); diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 7988e597c..4117ad961 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -50,6 +50,8 @@ public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES_PASSWORD = "password"; + private static final String KEY_ES_PARALLELISM = "parallelism"; + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -65,6 +67,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, IStreamSi private TypeInformation[] fieldTypes; - private int parallelism = -1; + private int parallelism = 1; + + private String registerTableName; private ElasticsearchTableInfo esTableInfo; @@ -121,7 +124,7 @@ private RichSinkFunction createEsSinkFunction() { @Override public void emitDataStream(DataStream> dataStream) { RichSinkFunction richSinkFunction = createEsSinkFunction(); - DataStreamSink streamSink = dataStream.addSink(richSinkFunction); + DataStreamSink streamSink = dataStream.addSink(richSinkFunction).name(registerTableName); if (parallelism > 0) { streamSink.setParallelism(parallelism); } @@ -136,6 +139,9 @@ public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) columnTypes = esTableInfo.getFieldTypes(); esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); String id = esTableInfo.getId(); + parallelism = Objects.isNull(esTableInfo.getParallelism()) ? + parallelism : esTableInfo.getParallelism(); + registerTableName = esTableInfo.getName(); if (!StringUtils.isEmpty(id)) { idIndexList = Arrays.stream(StringUtils.split(id, ",")).map(Integer::valueOf).collect(Collectors.toList()); diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 22c2b72bc..e1e26e1ac 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -49,6 +49,8 @@ public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_TRUE = "true"; + private static final String KEY_PARALLELISM = "parallelism"; + @Override protected boolean fieldNameNeedsUpperCase() { return false; @@ -64,6 +66,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, IStreamSinkGener< private String clientPrincipal; private String clientKeytabFile; - private int parallelism = -1; + private int parallelism = 1; public HbaseSink() { @@ -115,7 +115,7 @@ public void emitDataStream(DataStream> dataStream) { HbaseOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction).setParallelism(parallelism).name(registerTabName); } @Override diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 7234216a7..427f5772c 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -56,7 +56,7 @@ public abstract class AbstractKafkaSink implements RetractStreamTableSink, protected String[] partitionKeys; protected String sinkOperatorName; protected Properties properties; - protected int parallelism; + protected int parallelism = 1; protected String topic; protected String tableName; diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 4ad8947a8..58bf8bc78 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -57,7 +57,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map, Serializable, IStreamSinkGener { @@ -36,7 +37,7 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private Integer defaultSocketReadTimeoutMs; - private int parallelism = -1; + private int parallelism = 1; @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { @@ -47,6 +48,8 @@ public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.defaultSocketReadTimeoutMs = kuduTableInfo.getDefaultSocketReadTimeoutMs(); this.workerCount = kuduTableInfo.getWorkerCount(); this.writeMode = kuduTableInfo.getWriteMode(); + this.parallelism = Objects.isNull(kuduTableInfo.getParallelism()) ? + parallelism : kuduTableInfo.getParallelism(); return this; } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java index 3f34055ec..39c12a789 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoSink.java @@ -33,6 +33,8 @@ import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; +import java.util.Objects; + /** * Reason: * Date: 2018/11/6 @@ -48,6 +50,8 @@ public class MongoSink implements RetractStreamTableSink, IStreamSinkGener< protected String userName; protected String password; protected String database; + protected Integer parallelism = 1; + protected String registerTableName; public MongoSink() { // TO DO NOTHING @@ -61,6 +65,9 @@ public MongoSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.userName = mongoTableInfo.getUserName(); this.password = mongoTableInfo.getPassword(); this.database = mongoTableInfo.getDatabase(); + this.parallelism = Objects.isNull(mongoTableInfo.getParallelism()) ? + parallelism : mongoTableInfo.getParallelism(); + this.registerTableName = mongoTableInfo.getName(); return this; } @@ -77,7 +84,9 @@ public void emitDataStream(DataStream> dataStream) { MongoOutputFormat outputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction) + .setParallelism(parallelism) + .name(registerTableName); } @Override diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java index cc49a3ba8..f99e69725 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisSink.java @@ -33,6 +33,8 @@ import org.apache.flink.types.Row; import java.util.List; +import java.util.Objects; + /** * @author yanxi */ @@ -64,6 +66,10 @@ public class RedisSink implements RetractStreamTableSink, IStreamSinkGener< protected String masterName; + protected Integer parallelism = 1; + + protected String registerTableName; + public RedisSink(){ } @@ -82,6 +88,9 @@ public RedisSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.minIdle = redisTableInfo.getMinIdle(); this.masterName = redisTableInfo.getMasterName(); this.timeout = redisTableInfo.getTimeout(); + this.parallelism = Objects.isNull(redisTableInfo.getParallelism()) ? + parallelism : redisTableInfo.getParallelism(); + this.registerTableName = redisTableInfo.getName(); return this; } @@ -108,7 +117,7 @@ public void emitDataStream(DataStream> dataStream) { .setMasterName(this.masterName); RedisOutputFormat redisOutputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(redisOutputFormat); - dataStream.addSink(richSinkFunction); + dataStream.addSink(richSinkFunction).setParallelism(parallelism).name(registerTableName); } @Override diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 8961f7da9..91e49c5fc 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -50,11 +50,12 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); if (!StringUtils.isEmpty(primaryKeysStr)) { - List primaryKeysList = Lists.newArrayList(); primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); - redisTableInfo.setPrimaryKeys(primaryKeysList); } + redisTableInfo.setPrimaryKeys(primaryKeysList); + redisTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(RedisTableInfo.PARALLELISM_KEY.toLowerCase()))); return redisTableInfo; } From 3d7cbb46e8feee135b759c1eb4c6466a799a2f28 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 25 Sep 2020 11:22:28 +0800 Subject: [PATCH 316/378] [fix] fix when JOIN xxx then table not register error --- .../main/java/com/dtstack/flink/sql/util/TableUtils.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 2aeb06cf8..879b9204b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -232,6 +232,13 @@ public static String dealSelectResultWithJoinInfo(JoinInfo joinInfo, SqlSelect s queueInfo.offer(joinInfo.getLeftNode()); } + if (joinInfo.getLeftNode().getKind() == AS) { + SqlNode leftSqlNode = ((SqlBasicCall) joinInfo.getLeftNode()).getOperands()[0]; + if (leftSqlNode.getKind() == UNION) { + queueInfo.offer(joinInfo.getLeftNode()); + } + } + queueInfo.offer(joinInfo); } replaceFromNodeForJoin(joinInfo, sqlNode); From df6dc408ba92125eeed0ca5071c2df04201a3a84 Mon Sep 17 00:00:00 2001 From: chuixue Date: Sun, 27 Sep 2020 10:09:51 +0800 Subject: [PATCH 317/378] [30589][kafka-sink]remove kafka retract data in append mode --- .../com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 6 +++++- .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + .../java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java | 1 + 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 1d0c4d0e2..55d1fa7f8 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; import org.apache.commons.lang3.StringUtils; @@ -59,6 +60,7 @@ public abstract class AbstractKafkaSink implements RetractStreamTableSink, protected int parallelism; protected String topic; protected String tableName; + protected String updateMode; protected TableSchema schema; protected SinkFunction kafkaProducer; @@ -98,9 +100,11 @@ protected TableSchema buildTableSchema(String[] fieldNames, TypeInformation[] @Override public void emitDataStream(DataStream> dataStream) { + if (updateMode.equalsIgnoreCase(EUpdateMode.APPEND.name())) { + dataStream = dataStream.filter((Tuple2 record) -> record.f0); + } DataStream mapDataStream = dataStream .map((Tuple2 record) -> new CRow(record.f1, record.f0)) - .filter(x -> x.change()) .returns(getRowTypeInfo()) .setParallelism(parallelism); diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 632bb720e..3f92366b8 100644 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -36,6 +36,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafkaSinkTableInfo); this.tableName = kafkaSinkTableInfo.getName(); + this.updateMode = kafkaSinkTableInfo.getUpdateMode(); this.topic = kafkaSinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafkaSinkTableInfo); diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index d22be3d59..402b6ed9b 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -38,6 +38,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafka09SinkTableInfo); this.tableName = kafka09SinkTableInfo.getName(); + this.updateMode = kafka09SinkTableInfo.getUpdateMode(); this.topic = kafka09SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka09SinkTableInfo); diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index eea78e121..09a1bd781 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -40,6 +40,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafka10SinkTableInfo); this.tableName = kafka10SinkTableInfo.getName(); + this.updateMode = kafka10SinkTableInfo.getUpdateMode(); this.topic = kafka10SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka10SinkTableInfo); diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index ea45280c7..eee1a4ce5 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -41,6 +41,7 @@ public KafkaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { Properties kafkaProperties = getKafkaProperties(kafka11SinkTableInfo); this.tableName = kafka11SinkTableInfo.getName(); + this.updateMode = kafka11SinkTableInfo.getUpdateMode(); this.topic = kafka11SinkTableInfo.getTopic(); this.partitioner = Optional.of(new CustomerFlinkPartition<>()); this.partitionKeys = getPartitionKeys(kafka11SinkTableInfo); From 1a05ab141fd76031764e6c18006dfed042ed24ba Mon Sep 17 00:00:00 2001 From: dapeng Date: Sun, 27 Sep 2020 16:59:52 +0800 Subject: [PATCH 318/378] =?UTF-8?q?elasticsearch=20bulk=20=E7=9A=84?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99=E6=AD=BB=E4=BA=861?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/elasticsearch/ElasticsearchSink.java | 1 + .../elasticsearch/table/ElasticsearchSinkParser.java | 3 +++ .../elasticsearch/table/ElasticsearchTableInfo.java | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java index b7d9de6fc..602d4cfba 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/ElasticsearchSink.java @@ -135,6 +135,7 @@ public ElasticsearchSink genStreamSink(AbstractTargetTableInfo targetTableInfo) type = esTableInfo.getEsType(); columnTypes = esTableInfo.getFieldTypes(); esAddressList = Arrays.asList(esTableInfo.getAddress().split(",")); + this.bulkFlushMaxActions = esTableInfo.getBatchSize(); String id = esTableInfo.getId(); if (!StringUtils.isEmpty(id)) { diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java index 22c2b72bc..560872b05 100644 --- a/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchSinkParser.java @@ -47,6 +47,8 @@ public class ElasticsearchSinkParser extends AbstractTableParser { private static final String KEY_ES6_PASSWORD = "password"; + public static final String BATCH_SIZE_KEY = "batchSize"; + private static final String KEY_TRUE = "true"; @Override @@ -64,6 +66,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Sun, 27 Sep 2020 17:56:46 +0800 Subject: [PATCH 319/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=E5=AF=86?= =?UTF-8?q?=E7=A0=81=E4=B8=8D=E8=83=BD=E5=8C=85=E5=90=AB#=EF=BC=8C?= =?UTF-8?q?=EF=BC=9F=E7=AD=89=E7=89=B9=E6=AE=8A=E5=AD=97=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/redis/RedisAsyncReqRow.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index d4d1505f8..e0a348c38 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -21,7 +21,9 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import io.lettuce.core.KeyValue; +import io.lettuce.core.api.async.RedisAsyncCommands; import io.lettuce.core.api.async.RedisStringAsyncCommands; +import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -87,11 +89,6 @@ public void open(Configuration parameters) throws Exception { private void buildRedisClient(RedisSideTableInfo tableInfo){ String url = redisSideTableInfo.getUrl(); String password = redisSideTableInfo.getPassword(); - if (password != null){ - password = password + "@"; - } else { - password = ""; - } String database = redisSideTableInfo.getDatabase(); if (database == null){ database = "0"; @@ -99,25 +96,28 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ switch (RedisType.parse(tableInfo.getRedisType())){ case STANDALONE: StringBuilder redisUri = new StringBuilder(); - redisUri.append("redis://").append(password).append(url).append("/").append(database); + redisUri.append("redis://").append(url).append("/").append(database); redisClient = RedisClient.create(redisUri.toString()); connection = redisClient.connect(); async = connection.async(); + ((RedisAsyncCommands)async).auth(password); break; case SENTINEL: StringBuilder sentinelUri = new StringBuilder(); - sentinelUri.append("redis-sentinel://").append(password) + sentinelUri.append("redis-sentinel://") .append(url).append("/").append(database).append("#").append(redisSideTableInfo.getMasterName()); redisClient = RedisClient.create(sentinelUri.toString()); connection = redisClient.connect(); async = connection.async(); + ((RedisAsyncCommands)async).auth(password); break; case CLUSTER: StringBuilder clusterUri = new StringBuilder(); - clusterUri.append("redis://").append(password).append(url); + clusterUri.append("redis://").append(url); clusterClient = RedisClusterClient.create(clusterUri.toString()); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); + ((RedisAdvancedClusterAsyncCommands)async).auth(password); default: break; } From 09f97a9c9389e0bff756f3dd5aaea477832b68b8 Mon Sep 17 00:00:00 2001 From: zoudaokoulife Date: Tue, 29 Sep 2020 23:03:52 +0800 Subject: [PATCH 320/378] =?UTF-8?q?=E4=BF=AE=E6=94=B94.1.X=20=E5=AF=B9?= =?UTF-8?q?=E5=BA=94=E5=88=B0flink=E7=89=88=E6=9C=AC=E5=8F=B7=E5=88=B01.8.?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0496c6dd8..374ec3c31 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ UTF-8 - 1.8.3 + 1.8.1 From 64b681fb6f4396d879304d00e94165be21a87666 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 9 Oct 2020 10:08:47 +0800 Subject: [PATCH 321/378] remove add file statment --- .../dtstack/flink/sql/parser/SqlParser.java | 20 ++++++++ .../flink/sql/parser/SqlParserTest.java | 51 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index 23f8e4942..badcf8d9c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -28,8 +28,11 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Reason: @@ -51,6 +54,8 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } + private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+"); + /** * flink support sql syntax * CREATE TABLE sls_stream() with (); @@ -74,6 +79,7 @@ public static SqlTree parseSql(String sql) throws Exception { .replace("\t", " ").trim(); List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); + sqlArr = removeAddFileStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ @@ -154,4 +160,18 @@ public static SqlTree parseSql(String sql) throws Exception { return sqlTree; } + + /** + * remove add file with statment etc. add file /etc/krb5.conf; + */ + private static List removeAddFileStmt(List stmts) { + List cleanedStmts = new ArrayList<>(); + for (String stmt : stmts) { + Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt); + if(!matcher.matches()) { + cleanedStmts.add(stmt); + } + } + return cleanedStmts; + } } diff --git a/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java new file mode 100644 index 000000000..fdbb03a75 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.parser; + +import org.junit.Assert; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.util.ArrayList; +import java.util.List; + +/** + * @program: flink.sql + * @author: wuren + * @create: 2020/09/15 + **/ +public class SqlParserTest { + + @Test + public void testRemoveAddFileStmt() throws Exception { + List rawStmts = new ArrayList<>(); + String sql1 = " add file asdasdasd "; + String sql2 = " aDd fIle With asdasdasd "; + String sql3 = " INSERT INTO dwd_foo SELECT id, name FROM ods_foo"; + String sql4 = " ADD FILE asb "; + rawStmts.add(sql1); + rawStmts.add(sql2); + rawStmts.add(sql3); + rawStmts.add(sql4); + + List stmts = Whitebox.invokeMethod(SqlParser.class, "removeAddFileStmt", rawStmts); + Assert.assertEquals(stmts.get(0), sql3); + } + +} \ No newline at end of file From badc105562e054dc94664f1f03d019e36f1c93f1 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 9 Oct 2020 10:54:31 +0800 Subject: [PATCH 322/378] add unit test dependence --- pom.xml | 53 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 0496c6dd8..d10954d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -39,11 +39,56 @@ + + UTF-8 + 1.8.3 + 2.7.3 + 4.12 + 2.21.0 + 2.0.4 + 0.7.8 + - - UTF-8 - 1.8.3 - + + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + org.mockito + mockito-core + + + + + org.jacoco + org.jacoco.agent + runtime + test + ${jacoco.version} + + From a786cf8b622a729ccaac594b283a251a16ad1ab5 Mon Sep 17 00:00:00 2001 From: wuren Date: Fri, 9 Oct 2020 13:55:15 +0800 Subject: [PATCH 323/378] add unit test dependences --- pom.xml | 53 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 374ec3c31..d10954d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -39,11 +39,56 @@ + + UTF-8 + 1.8.3 + 2.7.3 + 4.12 + 2.21.0 + 2.0.4 + 0.7.8 + - - UTF-8 - 1.8.1 - + + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + org.mockito + mockito-core + + + + + org.jacoco + org.jacoco.agent + runtime + test + ${jacoco.version} + + From ea2d6431db113d0d7122edeeab0a55e6f7762705 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sat, 10 Oct 2020 12:42:28 +0800 Subject: [PATCH 324/378] resolve conflict --- .../java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 6acfcb760..1269b2844 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -101,10 +101,6 @@ public static String[] splitIgnoreQuotaBrackets(String str, String delimiter){ public ReplaceInfo getReplaceInfo(String field){ field = field.trim(); - if(field.length() <= 2){ - throw new RuntimeException(field + " \n" + - "Format defined exceptions"); - } //判断是不是常量==>''包裹的标识 if(field.startsWith("'") && field.endsWith("'")){ From 3f7488d0120439e0045e26388bfb4c6a19326f79 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sat, 10 Oct 2020 15:00:58 +0800 Subject: [PATCH 325/378] =?UTF-8?q?[fix]=20Flink=E4=B8=8D=E6=94=AF?= =?UTF-8?q?=E6=8C=81text=EF=BC=8C=E5=AF=B9=E7=94=A8=E6=88=B7=E5=81=9A?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index ec88bdbe9..21551364f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -68,9 +68,10 @@ public static Class stringConvertClass(String str) { case "varchar": case "char": - case "text": case "string": return String.class; + case "text": + throw new IllegalArgumentException(str + " type is not support, please use STRING. "); case "real": case "float": From c8a367483cb25dde581588281686a982264ef64e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sat, 10 Oct 2020 15:36:59 +0800 Subject: [PATCH 326/378] =?UTF-8?q?[fix]=20Flink=E4=B8=8D=E6=94=AF?= =?UTF-8?q?=E6=8C=81text=EF=BC=8C=E5=AF=B9=E7=94=A8=E6=88=B7=E5=81=9A?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../side/elasticsearch6/util/ClassUtil.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java index 90ac58229..4dd1c4300 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/ClassUtil.java @@ -15,12 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package com.dtstack.flink.sql.side.elasticsearch6.util; import com.dtstack.flink.sql.util.DateUtil; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.sql.Date; import java.sql.Time; @@ -30,13 +29,19 @@ * Reason: TODO ADD REASON(可选) * Date: 2017年03月10日 下午1:16:37 * Company: www.dtstack.com - * * @author sishu.yss */ public class ClassUtil { public static Class stringConvertClass(String str) { - switch (str.toLowerCase()) { + + // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型 + String lowerStr = str.toLowerCase().trim(); + if (lowerStr.startsWith("array")) { + return Array.newInstance(Integer.class, 0).getClass(); + } + + switch (lowerStr) { case "boolean": case "bit": return Boolean.class; @@ -54,7 +59,6 @@ public static Class stringConvertClass(String str) { case "blob": return Byte.class; - case "long": case "bigint": case "intunsigned": case "integerunsigned": @@ -63,8 +67,10 @@ public static Class stringConvertClass(String str) { case "varchar": case "char": - case "text": + case "string": return String.class; + case "text": + throw new IllegalArgumentException(str + " type is not support, please use STRING. "); case "real": case "float": @@ -90,9 +96,10 @@ public static Class stringConvertClass(String str) { case "decimalunsigned": return BigDecimal.class; default: - throw new RuntimeException("不支持 " + str + " 类型"); + break; } + throw new RuntimeException("不支持 " + str + " 类型"); } public static Object convertType(Object field, String fromType, String toType) { From 1284859eced6f93556b40feef8284d283fc82539 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 13 Oct 2020 09:58:15 +0800 Subject: [PATCH 327/378] rollback flink version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d10954d3e..d21ae0f9a 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ UTF-8 - 1.8.3 + 1.8.1 2.7.3 4.12 2.21.0 From 42c7c18c23d60ce783c5b257a23bf96797711814 Mon Sep 17 00:00:00 2001 From: dapeng Date: Tue, 13 Oct 2020 20:56:36 +0800 Subject: [PATCH 328/378] =?UTF-8?q?=E5=8D=87=E7=BA=A7flink=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E8=87=B31.8.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d21ae0f9a..d10954d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ UTF-8 - 1.8.1 + 1.8.3 2.7.3 4.12 2.21.0 From d87145c8fc582ec87665ff4ba103d532dbe2a372 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 21 Oct 2020 10:32:34 +0800 Subject: [PATCH 329/378] =?UTF-8?q?fix=20=E8=BF=9E=E6=8E=A5=E6=96=AD?= =?UTF-8?q?=E5=BC=80=E4=BB=BB=E5=8A=A1=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/rdb/writer/AbstractUpsertWriter.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index 42ed545a6..bc4f763a5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -170,8 +170,16 @@ public void executeUpdate(Connection connection) throws SQLException { connection.commit(); } catch (Exception e) { // deal pg error: current transaction is aborted, commands ignored until end of transaction block - connection.rollback(); - connection.commit(); + try { + connection.rollback(); + connection.commit(); + } catch (SQLException e1) { + throw new RuntimeException(e1); + } + + if(e.getMessage().contains("doesn't exist")){ + throw new RuntimeException(e); + } if (metricOutputFormat.outDirtyRecords.getCount() % DIRTYDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", entry.getValue()); LOG.error("", e); From ae223316115642e075a6db8169817d14c2603df2 Mon Sep 17 00:00:00 2001 From: dapeng Date: Wed, 21 Oct 2020 11:39:03 +0800 Subject: [PATCH 330/378] =?UTF-8?q?fix=20=E7=89=88=E6=9C=AC=E4=B8=8D?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E7=9A=84api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/redis/RedisAsyncReqRow.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index e0a348c38..c0d418fc1 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; import io.lettuce.core.KeyValue; +import io.lettuce.core.RedisURI; import io.lettuce.core.api.async.RedisAsyncCommands; import io.lettuce.core.api.async.RedisStringAsyncCommands; import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; @@ -95,29 +96,28 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } switch (RedisType.parse(tableInfo.getRedisType())){ case STANDALONE: - StringBuilder redisUri = new StringBuilder(); - redisUri.append("redis://").append(url).append("/").append(database); - redisClient = RedisClient.create(redisUri.toString()); + RedisURI redisURI = RedisURI.create("redis://" + url); + redisURI.setPassword(password); + redisURI.setDatabase(Integer.valueOf(database)); + redisClient = RedisClient.create(redisURI); connection = redisClient.connect(); async = connection.async(); - ((RedisAsyncCommands)async).auth(password); break; case SENTINEL: - StringBuilder sentinelUri = new StringBuilder(); - sentinelUri.append("redis-sentinel://") - .append(url).append("/").append(database).append("#").append(redisSideTableInfo.getMasterName()); - redisClient = RedisClient.create(sentinelUri.toString()); + RedisURI redisSentinelURI = RedisURI.create("redis-sentinel://" + url); + redisSentinelURI.setPassword(password); + redisSentinelURI.setDatabase(Integer.valueOf(database)); + redisSentinelURI.setSentinelMasterId(redisSideTableInfo.getMasterName()); + redisClient = RedisClient.create(redisSentinelURI); connection = redisClient.connect(); async = connection.async(); - ((RedisAsyncCommands)async).auth(password); break; case CLUSTER: - StringBuilder clusterUri = new StringBuilder(); - clusterUri.append("redis://").append(url); - clusterClient = RedisClusterClient.create(clusterUri.toString()); + RedisURI clusterURI = RedisURI.create("redis://" + url); + clusterURI.setPassword(password); + clusterClient = RedisClusterClient.create(clusterURI); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); - ((RedisAdvancedClusterAsyncCommands)async).auth(password); default: break; } From 44858b19b1190f2baf203ad3d9d4c8da4cb31938 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 26 Oct 2020 09:36:38 +0800 Subject: [PATCH 331/378] [feat] impala and kudu with kerberos [feat] impala support dynamic and static partition [feat] complete kudu with kerberos [fix] import and override error [feat] impala with kerberos --- .../sql/constrant/PluginParamConsts.java | 30 + .../dtstack/flink/sql/krb/KerberosTable.java | 65 ++ .../flink/sql/side/BaseAsyncReqRow.java | 7 +- .../com/dtstack/flink/sql/util/KrbUtils.java | 54 ++ .../dtstack/flink/sql/util/KrbUtilsTest.java | 44 + docs/plugin/kuduSide.md | 22 + docs/plugin/kuduSink.md | 149 +++- .../sql/side/impala/ImpalaAllReqRow.java | 37 +- .../sql/side/impala/ImpalaAsyncReqRow.java | 79 +- .../sql/sink/impala/ImpalaOutputFormat.java | 754 ++++++++++++++++++ .../flink/sql/sink/impala/ImpalaSink.java | 206 +++-- .../sink/impala/table/ImpalaSinkParser.java | 150 ++-- .../sink/impala/table/ImpalaTableInfo.java | 172 +++- .../flink/sql/side/kudu/KuduAllReqRow.java | 52 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 57 +- .../sql/side/kudu/table/KuduSideParser.java | 13 + .../side/kudu/table/KuduSideTableInfo.java | 51 +- .../flink/sql/sink/kudu/KuduOutputFormat.java | 52 +- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 20 +- .../sql/sink/kudu/table/KuduSinkParser.java | 13 + .../sql/sink/kudu/table/KuduTableInfo.java | 53 +- .../sql/side/rdb/async/RdbAsyncReqRow.java | 112 ++- .../sql/sink/rdb/JDBCTypeConvertUtils.java | 55 ++ .../rdb/format/JDBCUpsertOutputFormat.java | 16 +- 24 files changed, 1965 insertions(+), 298 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java create mode 100644 core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java create mode 100644 core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java create mode 100644 impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java b/core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java new file mode 100644 index 000000000..baf314c19 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/constrant/PluginParamConsts.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.constrant; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/15 + **/ +public class PluginParamConsts { + public static final String PRINCIPAL = "principal"; + public static final String KEYTAB = "keytab"; + public static final String KRB5_CONF = "krb5conf"; +} diff --git a/core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java b/core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java new file mode 100644 index 000000000..ce6691294 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/krb/KerberosTable.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.krb; + +import com.google.common.base.Strings; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/15 + **/ +public interface KerberosTable { + + String getPrincipal(); + + void setPrincipal(String principal); + + String getKeytab(); + + void setKeytab(String keytab); + + String getKrb5conf(); + + void setKrb5conf(String krb5conf); + + boolean isEnableKrb(); + + void setEnableKrb(boolean enableKrb); + + default void judgeKrbEnable() { + boolean allSet = + !Strings.isNullOrEmpty(getPrincipal()) && + !Strings.isNullOrEmpty(getKeytab()) && + !Strings.isNullOrEmpty(getKrb5conf()); + + boolean allNotSet = + Strings.isNullOrEmpty(getPrincipal()) && + Strings.isNullOrEmpty(getKeytab()) && + Strings.isNullOrEmpty(getKrb5conf()); + + if (allSet) { + setEnableKrb(true); + } else if (allNotSet) { + setEnableKrb(false); + } else { + throw new RuntimeException("Missing kerberos parameter! all kerberos params must be set, or all kerberos params are not set"); + } + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index ae8c75f7e..3b6a7f88c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -243,12 +243,7 @@ protected ScheduledFuture registerTimer(CRow input, ResultFuture result long timeoutTimestamp = sideInfo.getSideTableInfo().getAsyncTimeout() + getProcessingTimeService().getCurrentProcessingTime(); return getProcessingTimeService().registerTimer( timeoutTimestamp, - new ProcessingTimeCallback() { - @Override - public void onProcessingTime(long timestamp) throws Exception { - timeout(input, resultFuture); - } - }); + timestamp -> timeout(input, resultFuture)); } protected void cancelTimerWhenComplete(ResultFuture resultFuture, ScheduledFuture timerFuture){ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java new file mode 100644 index 000000000..2a83359c4 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/14 + **/ +public class KrbUtils { + + private static final Logger LOG = LoggerFactory.getLogger(KrbUtils.class); + + public static final String KRB5_CONF_KEY = "java.security.krb5.conf"; + public static final String HADOOP_AUTH_KEY = "hadoop.security.authentication"; + public static final String KRB_STR = "Kerberos"; +// public static final String FALSE_STR = "false"; +// public static final String SUBJECT_ONLY_KEY = "javax.security.auth.useSubjectCredsOnly"; + + public static UserGroupInformation getUgi(String principal, String keytabPath, String krb5confPath) throws IOException { + LOG.info("Kerberos login with principal: {} and keytab: {}", principal, keytabPath); + System.setProperty(KRB5_CONF_KEY, krb5confPath); + // TODO 尚未探索出此选项的意义，以后研究明白方可打开 +// System.setProperty(SUBJECT_ONLY_KEY, FALSE_STR); + Configuration configuration = new Configuration(); + configuration.set(HADOOP_AUTH_KEY , KRB_STR); + UserGroupInformation.setConfiguration(configuration); + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytabPath); + } + +} diff --git a/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java new file mode 100644 index 000000000..6c7359e14 --- /dev/null +++ b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +/** + * @program: flinkStreamSQL + * @author: wuren + * @create: 2020/09/14 + **/ +public class KrbUtilsTest { + @Test + public void testGetUgi() throws IOException { + String principal = ""; + String keytabPath = ""; + String krb5confPath = ""; + try { + KrbUtils.getUgi(principal, keytabPath, krb5confPath); + } catch (IllegalArgumentException e) { + Assert.assertEquals(e.getMessage(), "Can't get Kerberos realm"); + } + + } +} \ No newline at end of file diff --git a/docs/plugin/kuduSide.md b/docs/plugin/kuduSide.md index a2636d4aa..3075c3f35 100644 --- a/docs/plugin/kuduSide.md +++ b/docs/plugin/kuduSide.md @@ -60,6 +60,11 @@ | isFaultTolerant |查询是否容错 查询失败是否扫描第二个副本 默认false 容错 | 否|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| +| principal |kerberos用于登录的principal | 否|| +| keytab |keytab文件的路径 | 否|| +| krb5conf |conf文件路径 | 否|| +Kerberos三个参数全部设置则开启Kerberos认证，如果缺少任何一个则会提示缺少参数错误。 +如果全部未设置则不开启Kerberos连接Kudu集群。 -------------- ## 5.样例 @@ -163,3 +168,20 @@ into on t1.id = t2.id; ``` +## 7.kerberos示例 +``` +create table dim ( + name varchar, + id int, + PERIOD FOR SYSTEM_TIME +) WITH ( + type='kudu', + kuduMasters='host1', + tableName='foo', + parallelism ='1', + cache ='ALL', + keytab='foo/foobar.keytab', + krb5conf='bar/krb5.conf', + principal='kudu/host1@DTSTACK.COM' +); +``` \ No newline at end of file diff --git a/docs/plugin/kuduSink.md b/docs/plugin/kuduSink.md index cc8f8535a..7eb907770 100644 --- a/docs/plugin/kuduSink.md +++ b/docs/plugin/kuduSink.md @@ -11,7 +11,154 @@ CREATE TABLE tableName( writeMode='upsert', workerCount='1', defaultOperationTimeoutMs='600000', - defaultSocketReadTimeoutMs='6000000', + defaultSocketReadT## 1.格式： + ``` + CREATE TABLE tableName( + colName colType, + ... + colNameX colType + )WITH( + type ='kudu', + kuduMasters ='ip1,ip2,ip3', + tableName ='impala::default.test', + writeMode='upsert', + workerCount='1', + defaultOperationTimeoutMs='600000', + defaultSocketReadTimeoutMs='6000000', + parallelism ='parllNum' + ); + + + ``` + + ## 2.支持版本 + kudu 1.9.0+cdh6.2.0 + + ## 3.表结构定义 + + |参数名称|含义| + |----|---| + | tableName | 在 sql 中使用的名称;即注册到flink-table-env上的名称 + | colName | 列名称，redis中存储为 表名:主键名:主键值:列名]| + | colType | 列类型 [colType支持的类型](../colType.md)| + + + ## 4.参数： + + |参数名称|含义|是否必填|默认值| + |----|---|---|-----| + | type | 表名 输出表类型[mysq|hbase|elasticsearch|redis|kudu]|是|| + | kuduMasters | kudu master节点的地址;格式ip[ip，ip2]|是|| + | tableName | kudu 的表名称|是|| + | writeMode | 写入kudu的模式 insert|update|upsert |否 |upsert + | workerCount | 工作线程数 |否| + | defaultOperationTimeoutMs | 操作超时时间 |否| + | defaultSocketReadTimeoutMs | socket读取超时时间 |否| + | parallelism | 并行度设置|否|1| + | principal |kerberos用于登录的principal | 否|| + | keytab |keytab文件的路径 | 否|| + | krb5conf |conf文件路径 | 否|| + Kerberos三个参数全部设置则开启Kerberos认证，如果缺少任何一个则会提示缺少参数错误。 + 如果全部未设置则不开启Kerberos连接Kudu集群。 + + ## 5.样例： + ``` + CREATE TABLE MyTable( + channel varchar, + name varchar, + pv varchar, + a varchar, + b varchar + )WITH( + type ='kafka11', + bootstrapServers ='172.16.8.107:9092', + zookeeperQuorum ='172.16.8.107:2181/kafka', + offsetReset ='latest', + topic ='es_test', + timezone='Asia/Shanghai', + updateMode ='append', + enableKeyPartitions ='false', + topicIsPattern ='false', + parallelism ='1' + ); + + CREATE TABLE MyResult( + a string, + b string, + c string, + d string + )WITH( + type ='kudu', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='myresult', + writeMode='insert', + parallelism ='1' + ); + + CREATE TABLE sideTable( + c string, + d string, + PRIMARY KEY(c) , + PERIOD FOR SYSTEM_TIME + )WITH( + type ='kudu', + kuduMasters ='cdh03.cdhsite:7051', + tableName ='sidetest4', + partitionedJoin ='false', + cache ='LRU', + cacheSize ='10000', + cacheTTLMs ='60000', + parallelism ='1', + primaryKey ='c', + isFaultTolerant ='false' + ); + + insert + into + MyResult + select + MyTable.a, + MyTable.b, + s.c, + s.d + from + MyTable + join + sideTable s + on MyTable.a = s.c + where + MyTable.a='2' + and s.d='2' + + ``` + + ## 6.数据示例 + ### 输入数据 + ``` + {"channel":"daishuyun","name":"roc","pv":"10","a":"2","b":"2"} + ``` + ### 结果数据 + ``` + {"a":"2","b":"2","c":"3","d":"4"} + ``` + + ## 7.kerberos示例 + ``` + create table dwd ( + name varchar, + id int + ) WITH ( + type='kudu', + kuduMasters='host1', + tableName='foo', + writeMode='insert', + parallelism ='1', + keytab='foo/foobar.keytab', + krb5conf='bar/krb5.conf', + principal='kudu/host1@DTSTACK.COM' + ); + ``` +imeoutMs='6000000', parallelism ='parllNum' ); diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 10938308a..179e52e56 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -18,21 +18,23 @@ package com.dtstack.flink.sql.side.impala; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.all.AbstractRdbAllReqRow; import com.dtstack.flink.sql.util.JDBCUtils; +import com.dtstack.flink.sql.util.KrbUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; import java.util.List; /** @@ -61,10 +63,26 @@ public ImpalaAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List) () -> { + try { + return DriverManager.getConnection(url); + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + }); + } else { + connection = DriverManager.getConnection(url); + } connection.setAutoCommit(false); return connection; } catch (Exception e) { @@ -78,14 +96,11 @@ public String getUrl() throws IOException { String newUrl = ""; Integer authMech = impalaSideTableInfo.getAuthMech(); - StringBuffer urlBuffer = new StringBuffer(impalaSideTableInfo.getUrl()); + StringBuilder urlBuffer = new StringBuilder(impalaSideTableInfo.getUrl()); if (authMech == 0) { newUrl = urlBuffer.toString(); } else if (authMech == 1) { - String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); - String krb5FilePath = impalaSideTableInfo.getKrb5FilePath(); - String principal = impalaSideTableInfo.getPrincipal(); String krbRealm = impalaSideTableInfo.getKrbRealm(); String krbHostFQDN = impalaSideTableInfo.getKrbHostFQDN(); String krbServiceName = impalaSideTableInfo.getKrbServiceName(); @@ -96,11 +111,7 @@ public String getUrl() throws IOException { .concat("KrbServiceName=").concat(krbServiceName).concat(";") ); newUrl = urlBuffer.toString(); - System.setProperty("java.security.krb5.conf", krb5FilePath); - Configuration configuration = new Configuration(); - configuration.set("hadoop.security.authentication" , "Kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(principal, keyTabFilePath); + } else if (authMech == 2) { String uName = impalaSideTableInfo.getUserName(); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index 3f4817580..ae8ac2be9 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -18,27 +18,32 @@ package com.dtstack.flink.sql.side.impala; -import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.impala.table.ImpalaSideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; +import com.dtstack.flink.sql.util.KrbUtils; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonObject; import io.vertx.ext.jdbc.JDBCClient; +import io.vertx.ext.sql.SQLClient; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; +import java.security.PrivilegedAction; import java.util.List; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** * Date: 2019/11/12 @@ -53,6 +58,7 @@ public class ImpalaAsyncReqRow extends RdbAsyncReqRow { private final static String IMPALA_DRIVER = "com.cloudera.impala.jdbc41.Driver"; + protected UserGroupInformation ugi = null; public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new ImpalaAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -60,9 +66,19 @@ public ImpalaAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List inputParams, + CRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { + if (ugi == null) { + doAsyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); + } else { + // Kerberos + ugi.doAs((PrivilegedAction) () -> { + doAsyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); + return null; + }); + } + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java new file mode 100644 index 000000000..19e904ef2 --- /dev/null +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -0,0 +1,754 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.sink.impala; + +import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; +import com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils; +import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.util.JDBCUtils; +import com.dtstack.flink.sql.util.KrbUtils; +import com.google.common.collect.Maps; +import org.apache.commons.collections.CollectionUtils; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.types.Row; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.rmi.RemoteException; +import java.security.PrivilegedExceptionAction; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils.setRecordToStatement; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Date: 2020/10/14 + * Company: www.dtstack.com + * + * @author tiezhu + */ +public class ImpalaOutputFormat extends AbstractDtRichOutputFormat> { + + private static final Logger LOG = LoggerFactory.getLogger(ImpalaOutputFormat.class); + + private static final long serialVersionUID = 1L; + + // ${field} + private static final Pattern STATIC_PARTITION_PATTERN = Pattern.compile("\\$\\{([^}]*)}"); + // cast(value as string) -> cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) + private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); + //specific type which values need to be quoted + private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp"}; + + private static final Integer DEFAULT_CONN_TIME_OUT = 60; + private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; + private static final int DIRTY_DATA_PRINT_FREQUENCY = 1000; + + private static final String KUDU_TYPE = "kudu"; + private static final String UPDATE_MODE = "update"; + private static final String PARTITION_CONSTANT = "PARTITION"; + private static final String DRIVER_NAME = "com.cloudera.impala.jdbc41.Driver"; + + private static final String VALUES_CONDITION = "${valuesCondition}"; + private static final String PARTITION_CONDITION = "${partitionCondition}"; + private static final String TABLE_FIELDS_CONDITION = "${tableFieldsCondition}"; + private static final String NO_PARTITION = "noPartition"; + + protected transient Connection connection; + protected transient Statement statement; + protected transient PreparedStatement updateStatement; + + private transient volatile boolean closed = false; + private int batchCount = 0; + + // |------------------------------------------------| + // | partitionCondition |Array of valueCondition| + // |------------------------------------------------| + // | ptOne, ptTwo, ptThree | [(v1, v2, v3, v4, v5)]| DP + // |------------------------------------------------| + // | ptOne = v1, ptTwo = v2 | [(v3, v4, v5)] | SP + // |------------------------------------------------| + // | ptOne, ptTwo = v2 | [(v1, v3, v4, v5)] | DP and SP + // |------------------------------------------------| + // | noPartition | [(v1, v2, v3, v4, v5)]| kudu or disablePartition + // |------------------------------------------------| + private transient Map> rowDataMap; + + protected String keytabPath; + protected String krb5confPath; + protected String principal; + protected Integer authMech; + protected String dbUrl; + protected String userName; + protected String password; + protected int batchSize = 100; + protected long batchWaitInterval = 60 * 1000L; + protected String tableName; + protected List primaryKeys; + protected String partitionFields; + protected Boolean enablePartition; + protected String schema; + protected String storeType; + protected String updateMode; + public List fieldNames; + public List fieldTypes; + public List fieldExtraInfoList; + + // partition field of static partition which matched by ${field} + private final List staticPartitionFields = new ArrayList<>(); + + // valueFieldsName -> 重组之后的fieldNames，为了重组row data字段值对应 + // 需要对partition字段做特殊处理，比如原来的字段顺序为(age, name, id)，但是因为partition，写入的SQL为 + // INSERT INTO tableName(name, id) PARTITION(age) VALUES(?, ?, ?) + // 那么实际executeSql设置字段的顺序应该为(name, id, age)，同时，字段对应的type顺序也需要重组 + private List valueFieldNames; + private transient AbstractDtRichOutputFormat metricOutputFormat; + private List rows; + + private transient ScheduledExecutorService scheduler; + private transient ScheduledFuture scheduledFuture; + + @Override + public void configure(Configuration parameters) { + } + + @Override + public void open(int taskNumber, int numTasks) throws IOException { + try { + rowDataMap = new HashMap<>(); + rows = new ArrayList<>(); + metricOutputFormat = this; + openConnect(); + initScheduledTask(batchWaitInterval); + init(); + initMetric(); + } catch (Exception e) { + throw new RemoteException("impala output format open error!", e); + } + } + + private void init() throws SQLException { + if (Objects.nonNull(partitionFields)) { + // match ${field} from partitionFields + Matcher matcher = STATIC_PARTITION_PATTERN.matcher(partitionFields); + while (matcher.find()) { + LOG.info("find static partition field: {}", matcher.group(1)); + staticPartitionFields.add(matcher.group(1)); + } + } + + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + if (!storeType.equalsIgnoreCase(KUDU_TYPE)) { + throw new IllegalArgumentException("update mode not support for non-kudu table!"); + } + + updateStatement = connection.prepareStatement(buildUpdateSql(schema, tableName, fieldNames, primaryKeys)); + return; + } + + valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields); + } + + private void initScheduledTask(Long batchWaitInterval) { + if (batchWaitInterval != 0) { + this.scheduler = new ScheduledThreadPoolExecutor(1, + new DTThreadFactory("impala-upsert-output-format")); + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { + synchronized (ImpalaOutputFormat.this) { + try { + flush(); + } catch (Exception e) { + LOG.error("Writing records to impala jdbc failed.", e); + throw new RuntimeException("Writing records to impala jdbc failed.", e); + } + } + }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + } + } + + private void openConnect() throws IOException { + if (authMech == 1) { + UserGroupInformation ugi = KrbUtils.getUgi(principal, keytabPath, krb5confPath); + try { + ugi.doAs((PrivilegedExceptionAction) () -> { + openJdbc(); + return null; + }); + } catch (InterruptedException | IOException e) { + throw new IllegalArgumentException("connect impala error!", e); + } + } else { + openJdbc(); + } + } + + /** + * get jdbc connection + */ + private void openJdbc() { + JDBCUtils.forName(DRIVER_NAME, getClass().getClassLoader()); + try { + connection = DriverManager.getConnection(dbUrl, userName, password); + statement = connection.createStatement(); + connection.setAutoCommit(false); + } catch (SQLException sqlException) { + throw new RuntimeException("get impala jdbc connection failed!", sqlException); + } + } + + private synchronized void flush() throws SQLException { + if (batchCount > 0) { + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + executeUpdateBatch(); + } + if (!rowDataMap.isEmpty()) { + String templateSql = + "INSERT INTO tableName ${tableFieldsCondition} PARTITION ${partitionCondition} VALUES ${valuesCondition}"; + executeBatchSql( + statement, + templateSql, + schema, + tableName, + storeType, + enablePartition, + valueFieldNames, + partitionFields, + rowDataMap + ); + rowDataMap.clear(); + } + } + batchCount = 0; + + } + + /** + * execute batch update statement + * + * @throws SQLException throw sql exception + */ + private void executeUpdateBatch() throws SQLException { + try { + rows.forEach(row -> { + try { + JDBCTypeConvertUtils.setRecordToStatement( + updateStatement, + JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), + row, + primaryKeys.stream().mapToInt(fieldNames::indexOf).toArray() + ); + updateStatement.addBatch(); + } catch (Exception e) { + throw new RuntimeException("impala jdbc execute batch error!", e); + } + }); + updateStatement.executeBatch(); + connection.commit(); + rows.clear(); + } catch (Exception e) { + LOG.debug("impala jdbc execute batch error ", e); + connection.rollback(); + connection.commit(); + updateStatement.clearBatch(); + executeUpdate(connection); + } + } + + public void executeUpdate(Connection connection) { + rows.forEach(row -> { + try { + setRecordToStatement(updateStatement, JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), row); + updateStatement.executeUpdate(); + connection.commit(); + } catch (Exception e) { + try { + connection.rollback(); + connection.commit(); + } catch (SQLException e1) { + throw new RuntimeException(e1); + } + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTY_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.error("record insert failed ,this row is {}", row.toString()); + LOG.error("", e); + } + metricOutputFormat.outDirtyRecords.inc(); + } + }); + rows.clear(); + } + + private void putRowIntoMap(Map> rowDataMap, Tuple2 rowData) { + Set keySet = rowDataMap.keySet(); + ArrayList tempRowArray; + if (keySet.contains(rowData.f0)) { + tempRowArray = rowDataMap.get(rowData.f0); + } else { + tempRowArray = new ArrayList<>(); + } + tempRowArray.add(rowData.f1); + rowDataMap.put(rowData.f0, tempRowArray); + } + + private List rebuildFieldNameListAndTypeList(List fieldNames, + List staticPartitionFields, + List fieldTypes, + String partitionFields) { + if (partitionFields.isEmpty()) { + return fieldNames; + } + + List valueFields = new ArrayList<>(fieldNames); + + for (int i = valueFields.size() - 1; i >= 0; i--) { + if (staticPartitionFields.contains(fieldNames.get(i))) { + valueFields.remove(i); + fieldTypes.remove(i); + } + } + + for (int i = 0; i < valueFields.size(); i++) { + if (partitionFields.contains(fieldNames.get(i))) { + valueFields.add(valueFields.remove(i)); + fieldTypes.add(fieldTypes.remove(i)); + } + } + + return valueFields; + } + + /** + * Quote a specific type of value, like string, timestamp + * before: 1, cast(tiezhu as string), cast(2001-01-09 01:05:01 as timestamp), cast(123 as int) + * after: 1, cast('tiezhu' as string), cast('2001-01-09 01:05:01' as timestamp), cast(123 as int) + * if cast value is null, then cast(null as type) + * + * @param valueCondition original value condition + * @return quoted condition + */ + private String valueConditionAddQuotation(String valueCondition) { + final String[] valueConditionCopy = {valueCondition}; + String[] temps = valueCondition.split(","); + Arrays.stream(temps).forEach( + item -> { + Matcher matcher = TYPE_PATTERN.matcher(item); + while (matcher.find()) { + String value = matcher.group(1); + String type = matcher.group(2); + + if (Arrays.asList(NEED_QUOTE_TYPE).contains(type)) { + if (!"null".equals(value)) { + valueConditionCopy[0] = valueConditionCopy[0].replace(value, "'" + value + "'"); + } + } + } + } + ); + return "(" + valueConditionCopy[0] + ")"; + } + + @Override + public synchronized void writeRecord(Tuple2 record) throws IOException { + try { + if (!record.f0) { + return; + } + + if (outRecords.getCount() % RECEIVE_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.info("Receive data : {}", record); + } + + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + rows.add(Row.copy(record.f1)); + } else { + Map valueMap = Maps.newHashMap(); + Row row = Row.copy(record.f1); + + for (int i = 0; i < row.getArity(); i++) { + valueMap.put(fieldNames.get(i), row.getField(i)); + } + + Tuple2 rowTuple2 = new Tuple2<>(); + if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { + rowTuple2.f0 = NO_PARTITION; + } else { + rowTuple2.f0 = buildPartitionCondition(valueMap, partitionFields, staticPartitionFields); + } + + // 根据字段名对 row data 重组, 比如，原始 row data : (1, xxx, 20) -> (id, name, age) + // 但是由于 partition，写入的field 顺序变成了 (name, id, age)，则需要对 row data 重组变成 (xxx, 1, 20) + Row rowValue = new Row(fieldTypes.size()); + for (int i = 0; i < fieldTypes.size(); i++) { + rowValue.setField(i, valueMap.get(valueFieldNames.get(i))); + } + rowTuple2.f1 = valueConditionAddQuotation(buildValuesCondition(fieldTypes, rowValue)); + putRowIntoMap(rowDataMap, rowTuple2); + } + + batchCount++; + + if (batchCount >= batchSize) { + flush(); + } + + // Receive data + outRecords.inc(); + } catch (Exception e) { + throw new RuntimeException("Writing records to impala failed.", e); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + // 将还未执行的SQL flush + if (batchCount > 0) { + try { + flush(); + } catch (Exception e) { + throw new RuntimeException("Writing records to impala failed.", e); + } + } + // cancel scheduled task + if (this.scheduledFuture != null) { + scheduledFuture.cancel(false); + this.scheduler.shutdown(); + } + // close connection + try { + if (connection != null && connection.isValid(DEFAULT_CONN_TIME_OUT)) { + connection.close(); + } + + if (statement != null && !statement.isClosed()) { + statement.close(); + } + + if (updateStatement != null && !updateStatement.isClosed()) { + updateStatement.close(); + } + } catch (SQLException e) { + throw new RemoteException("impala connection close failed!"); + } finally { + connection = null; + statement = null; + updateStatement = null; + } + closed = true; + } + + /** + * execute batch sql from row data map + * sql like 'insert into tableName(f1, f2, f3) ${partitionCondition} values(v1, v2, v3), (v4, v5, v6).... + * + * @param statement execute statement + * @param tempSql template sql + * @param storeType the store type of data + * @param enablePartition enable partition or not + * @param fieldNames field name list + * @param partitionFields partition fields + * @param rowDataMap row data map + * @throws SQLException throw sql exception + */ + private synchronized void executeBatchSql(Statement statement, + String tempSql, + String schema, + String tableName, + String storeType, + Boolean enablePartition, + List fieldNames, + String partitionFields, + Map> rowDataMap) throws SQLException { + StringBuilder valuesCondition = new StringBuilder(); + StringBuilder partitionCondition = new StringBuilder(); + String tableFieldsCondition = buildTableFieldsCondition(fieldNames, partitionFields); + ArrayList rowData; + String tableNameInfo = Objects.isNull(schema) ? + tableName : quoteIdentifier(schema) + "." + tableName; + tempSql = tempSql.replace("tableName", tableNameInfo); + + // kudu ${partitionCondition} is null + if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { + rowData = rowDataMap.get(NO_PARTITION); + rowData.forEach(row -> valuesCondition.append(row).append(", ")); + String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) + .replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(PARTITION_CONSTANT, "") + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); + String substring = executeSql.substring(0, executeSql.length() - 2); + statement.execute(substring); + return; + } + + // partition sql + Set keySet = rowDataMap.keySet(); + String finalTempSql = tempSql; + keySet.forEach(key -> { + try { + String executeSql = String.copyValueOf(finalTempSql.toCharArray()); + ArrayList valuesConditionList = rowDataMap.get(key); + partitionCondition.append(key); + executeSql = executeSql.replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition) + .replace(VALUES_CONDITION, String.join(", ", valuesConditionList)); + statement.execute(executeSql); + partitionCondition.delete(0, partitionCondition.length()); + } catch (SQLException sqlException) { + throw new RuntimeException("execute impala partition SQL error! ", sqlException); + } + }); + } + + /** + * build partition condition with row data + * + * @param rowData row data + * @param partitionFields partition fields + * @param staticPartitionField static partition fields + * @return condition like '(ptOne, ptTwo=v2)' + */ + private String buildPartitionCondition(Map rowData, String partitionFields, List staticPartitionField) { + for (String key : staticPartitionField) { + StringBuilder sb = new StringBuilder(); + Object value = rowData.get(key); + sb.append(key).append("=").append(value); + partitionFields = partitionFields.replace("${" + key + "}", sb.toString()); + } + return "(" + partitionFields + ")"; + } + + /** + * build field condition according to field names + * replace ${tableFieldCondition} + * + * @param fieldNames the selected field names + * @param partitionFields the partition fields + * @return condition like '(id, name, age)' + */ + private String buildTableFieldsCondition(List fieldNames, String partitionFields) { + return "(" + fieldNames.stream() + .filter(f -> !partitionFields.contains(f)) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")) + ")"; + } + + /** + * according to field types, build the values condition + * replace ${valuesCondition} + * + * @param fieldTypes field types + * @return condition like '(?, ?, cast(? as string))' and '?' will be replaced with row data + */ + private String buildValuesCondition(List fieldTypes, Row row) { + String valuesCondition = fieldTypes.stream().map( + f -> { + if (Arrays.asList(NEED_QUOTE_TYPE).contains(f.toLowerCase())) { + return String.format("cast(? as %s)", f.toLowerCase()); + } + return "?"; + }).collect(Collectors.joining(", ")); + for (int i = 0; i < row.getArity(); i++) { + valuesCondition = valuesCondition.replaceFirst("\\?", Objects.isNull(row.getField(i)) ? "null" : row.getField(i).toString()); + } + return valuesCondition; + } + + /** + * impala update mode SQL + * + * @return UPDATE tableName SET setCondition WHERE whereCondition + */ + private String buildUpdateSql(String schema, String tableName, List fieldNames, List primaryKeys) { + //跳过primary key字段 + String setClause = fieldNames.stream() + .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); + + String conditionClause = primaryKeys.stream() + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); + + return "UPDATE " + (Objects.isNull(schema) ? "" : quoteIdentifier(schema) + ".") + + quoteIdentifier(tableName) + " SET " + setClause + " WHERE " + conditionClause; + } + + private String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + public static Builder getImpalaBuilder() { + return new Builder(); + } + + public static class Builder { + private final ImpalaOutputFormat format = new ImpalaOutputFormat(); + + public Builder setDbUrl(String dbUrl) { + format.dbUrl = dbUrl; + return this; + } + + public Builder setUserName(String userName) { + format.userName = userName; + return this; + } + + public Builder setPassword(String password) { + format.password = password; + return this; + } + + public Builder setBatchSize(Integer batchSize) { + format.batchSize = batchSize; + return this; + } + + public Builder setBatchWaitInterval(Long batchWaitInterval) { + format.batchWaitInterval = batchWaitInterval; + return this; + } + + public Builder setTableName(String tableName) { + format.tableName = tableName; + return this; + } + + public Builder setPartitionFields(String partitionFields) { + format.partitionFields = Objects.isNull(partitionFields) ? + "" : partitionFields; + return this; + } + + public Builder setPrimaryKeys(List primaryKeys) { + format.primaryKeys = primaryKeys; + return this; + } + + public Builder setSchema(String schema) { + format.schema = schema; + return this; + } + + public Builder setEnablePartition(Boolean enablePartition) { + format.enablePartition = enablePartition; + return this; + } + + public Builder setUpdateMode(String updateMode) { + format.updateMode = updateMode; + return this; + } + + public Builder setFieldList(List fieldList) { + format.fieldNames = fieldList; + return this; + } + + public Builder setFieldTypeList(List fieldTypeList) { + format.fieldTypes = fieldTypeList; + return this; + } + + public Builder setStoreType(String storeType) { + format.storeType = storeType; + return this; + } + + public Builder setFieldExtraInfoList(List fieldExtraInfoList) { + format.fieldExtraInfoList = fieldExtraInfoList; + return this; + } + + public Builder setKeyTabPath(String keyTabPath) { + format.keytabPath = keyTabPath; + return this; + } + + public Builder setKrb5ConfPath(String krb5ConfPath) { + format.krb5confPath = krb5ConfPath; + return this; + } + + public Builder setPrincipal(String principal) { + format.principal = principal; + return this; + } + + public Builder setAuthMech(Integer authMech) { + format.authMech = authMech; + return this; + } + + private boolean canHandle(String url) { + return url.startsWith("jdbc:impala:"); + } + + public ImpalaOutputFormat build() { + if (!canHandle(format.dbUrl)) { + throw new IllegalArgumentException("impala dbUrl is illegal, check url: " + format.dbUrl); + } + + if (format.authMech == EAuthMech.Kerberos.getType()) { + checkNotNull(format.krb5confPath, + "When kerberos authentication is enabled, krb5confPath is required！"); + checkNotNull(format.principal, + "When kerberos authentication is enabled, principal is required！"); + checkNotNull(format.keytabPath, + "When kerberos authentication is enabled, keytabPath is required！"); + } + + if (format.authMech == EAuthMech.UserName.getType()) { + checkNotNull(format.userName, "userName is required!"); + } + + if (format.authMech == EAuthMech.NameANDPassword.getType()) { + checkNotNull(format.userName, "userName is required!"); + checkNotNull(format.password, "password is required!"); + } + + return format; + } + + } + +} diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java index 34101d410..6d28f0173 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaSink.java @@ -20,64 +20,115 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.impala.table.ImpalaTableInfo; -import com.dtstack.flink.sql.sink.rdb.JDBCOptions; -import com.dtstack.flink.sql.sink.rdb.AbstractRdbSink; -import com.dtstack.flink.sql.sink.rdb.format.JDBCUpsertOutputFormat; +import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.table.AbstractTargetTableInfo; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.types.Row; -import java.io.IOException; +import java.util.List; +import java.util.Objects; /** - * Date: 2019/11/11 + * Date: 2020/10/14 * Company: www.dtstack.com * - * @author xiuzhu + * @author tiezhu */ +public class ImpalaSink implements RetractStreamTableSink, IStreamSinkGener { -public class ImpalaSink extends AbstractRdbSink implements IStreamSinkGener { + private static final String DEFAULT_STORE_TYPE = "kudu"; - private ImpalaTableInfo impalaTableInfo; + protected String[] fieldNames; + TypeInformation[] fieldTypes; + protected String dbUrl; + protected String userName; + protected String password; + protected Integer authMech; + + protected String keytabPath; + protected String krb5confPath; + protected String principal; + + protected int batchSize = 100; + protected long batchWaitInterval = 60 * 1000L; + protected String tableName; + protected String registerTabName; + protected String storeType; + + protected List primaryKeys; + private int parallelism = 1; + protected String schema; + protected String updateMode; + protected Boolean enablePartition; + public List fieldList; + public List fieldTypeList; + public List fieldExtraInfoList; + protected String partitionFields; public ImpalaSink() { - super(null); + // do Nothing } @Override - public JDBCUpsertOutputFormat getOutputFormat() { - JDBCOptions jdbcOptions = JDBCOptions.builder() - .setDbUrl(getImpalaJdbcUrl()) - .setDialect(new ImpalaDialect(getFieldTypes(), primaryKeys)) - .setUsername(userName) - .setPassword(password) - .setTableName(tableName) - .build(); + public ImpalaSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { + ImpalaTableInfo impalaTableInfo = (ImpalaTableInfo) targetTableInfo; + this.dbUrl = getImpalaJdbcUrl(impalaTableInfo); + this.password = impalaTableInfo.getPassword(); + this.userName = impalaTableInfo.getUserName(); + this.authMech = impalaTableInfo.getAuthMech(); - return JDBCUpsertOutputFormat.builder() - .setOptions(jdbcOptions) - .setFieldNames(fieldNames) - .setFlushMaxSize(batchNum) - .setFlushIntervalMills(batchWaitInterval) - .setFieldTypes(sqlTypes) - .setKeyFields(primaryKeys) - .setPartitionFields(impalaTableInfo.getPartitionFields()) - .setAllReplace(allReplace) - .setUpdateMode(updateMode) - .build(); - } + this.principal = impalaTableInfo.getPrincipal(); + this.keytabPath = impalaTableInfo.getKeyTabFilePath(); + this.krb5confPath = impalaTableInfo.getKrb5FilePath(); + + this.updateMode = Objects.isNull(impalaTableInfo.getUpdateMode()) ? + "append" : impalaTableInfo.getUpdateMode(); + + this.batchSize = Objects.isNull(impalaTableInfo.getBatchSize()) ? + batchSize : impalaTableInfo.getBatchSize(); + this.batchWaitInterval = Objects.isNull(impalaTableInfo.getBatchWaitInterval()) ? + batchWaitInterval : impalaTableInfo.getBatchWaitInterval(); + this.parallelism = Objects.isNull(impalaTableInfo.getParallelism()) ? + parallelism : impalaTableInfo.getParallelism(); + this.registerTabName = impalaTableInfo.getTableName(); + this.fieldList = impalaTableInfo.getFieldList(); + this.fieldTypeList = impalaTableInfo.getFieldTypeList(); + this.fieldExtraInfoList = impalaTableInfo.getFieldExtraInfoList(); + this.tableName = impalaTableInfo.getTableName(); + this.schema = impalaTableInfo.getSchema(); + this.primaryKeys = impalaTableInfo.getPrimaryKeys(); + this.partitionFields = impalaTableInfo.getPartitionFields(); - public String getImpalaJdbcUrl() { + this.storeType = Objects.isNull(impalaTableInfo.getStoreType()) ? + DEFAULT_STORE_TYPE : impalaTableInfo.getStoreType(); + this.enablePartition = impalaTableInfo.isEnablePartition(); + + return this; + } + + /** + * build Impala Jdbc Url according to authMech + * + * @param impalaTableInfo impala table info + * @return jdbc url with auth mech info + */ + public String getImpalaJdbcUrl(ImpalaTableInfo impalaTableInfo) { Integer authMech = impalaTableInfo.getAuthMech(); - String newUrl = dbUrl; - StringBuffer urlBuffer = new StringBuffer(dbUrl); + String newUrl = impalaTableInfo.getUrl(); + StringBuilder urlBuffer = new StringBuilder(impalaTableInfo.getUrl()); if (authMech == EAuthMech.NoAuthentication.getType()) { return newUrl; } else if (authMech == EAuthMech.Kerberos.getType()) { - String keyTabFilePath = impalaTableInfo.getKeyTabFilePath(); - String krb5FilePath = impalaTableInfo.getKrb5FilePath(); - String principal = impalaTableInfo.getPrincipal(); String krbRealm = impalaTableInfo.getKrbRealm(); String krbHostFqdn = impalaTableInfo.getKrbHostFQDN(); String krbServiceName = impalaTableInfo.getKrbServiceName(); @@ -88,21 +139,10 @@ public String getImpalaJdbcUrl() { .concat("KrbServiceName=").concat(krbServiceName).concat(";") ); newUrl = urlBuffer.toString(); - - System.setProperty("java.security.krb5.conf", krb5FilePath); - Configuration configuration = new Configuration(); - configuration.set("hadoop.security.authentication", "Kerberos"); - UserGroupInformation.setConfiguration(configuration); - try { - UserGroupInformation.loginUserFromKeytab(principal, keyTabFilePath); - } catch (IOException e) { - throw new RuntimeException("loginUserFromKeytab error ..", e); - } - } else if (authMech == EAuthMech.UserName.getType()) { urlBuffer.append(";" .concat("AuthMech=3;") - .concat("UID=").concat(userName).concat(";") + .concat("UID=").concat(impalaTableInfo.getUserName()).concat(";") .concat("PWD=;") .concat("UseSasl=0") ); @@ -110,8 +150,8 @@ public String getImpalaJdbcUrl() { } else if (authMech == EAuthMech.NameANDPassword.getType()) { urlBuffer.append(";" .concat("AuthMech=3;") - .concat("UID=").concat(userName).concat(";") - .concat("PWD=").concat(password) + .concat("UID=").concat(impalaTableInfo.getUserName()).concat(";") + .concat("PWD=").concat(impalaTableInfo.getPassword()) ); newUrl = urlBuffer.toString(); } else { @@ -120,11 +160,69 @@ public String getImpalaJdbcUrl() { return newUrl; } + private ImpalaOutputFormat buildImpalaOutputFormat() { + + return ImpalaOutputFormat.getImpalaBuilder() + .setDbUrl(dbUrl) + .setPassword(password) + .setUserName(userName) + .setSchema(schema) + .setTableName(tableName) + .setUpdateMode(updateMode) + .setBatchSize(batchSize) + .setBatchWaitInterval(batchWaitInterval) + .setPrimaryKeys(primaryKeys) + .setPartitionFields(partitionFields) + .setFieldList(fieldList) + .setFieldTypeList(fieldTypeList) + .setFieldExtraInfoList(fieldExtraInfoList) + .setStoreType(storeType) + .setEnablePartition(enablePartition) + .setUpdateMode(updateMode) + .setAuthMech(authMech) + .setKeyTabPath(keytabPath) + .setKrb5ConfPath(krb5confPath) + .setPrincipal(principal) + .build(); + } + + @Override + public void emitDataStream(DataStream> dataStream) { + consumeDataStream(dataStream); + } + + public void consumeDataStream(DataStream> dataStream) { + ImpalaOutputFormat outputFormat = buildImpalaOutputFormat(); + RichSinkFunction> richSinkFunction = new OutputFormatSinkFunction(outputFormat); + dataStream.addSink(richSinkFunction) + .setParallelism(parallelism) + .name(tableName); + } + @Override - public AbstractRdbSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { - super.genStreamSink(targetTableInfo); - this.impalaTableInfo = (ImpalaTableInfo) targetTableInfo; + public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; return this; } + @Override + public TupleTypeInfo> getOutputType() { + return new TupleTypeInfo<>(org.apache.flink.table.api.Types.BOOLEAN(), getRecordType()); + } + + @Override + public TypeInformation getRecordType() { + return new RowTypeInfo(fieldTypes, fieldNames); + } + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public TypeInformation[] getFieldTypes() { + return fieldTypes; + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index 0d4857809..f1a475f61 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -18,109 +18,125 @@ package com.dtstack.flink.sql.sink.impala.table; -import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.util.MathUtil; -import java.math.BigDecimal; -import java.sql.Timestamp; import java.util.Arrays; import java.util.List; import java.util.Map; /** - * Reason: - * Date: 2019/11/11 + * Date: 2020/10/14 * Company: www.dtstack.com * - * @author xiuzhu + * @author tiezhu */ +public class ImpalaSinkParser extends AbstractTableParser { -public class ImpalaSinkParser extends RdbSinkParser { + private static final String PARALLELISM_KEY = "parallelism"; + + private static final String AUTH_MECH_KEY = "authMech"; + + private static final String KRB5FILEPATH_KEY = "krb5FilePath"; + + private static final String PRINCIPAL_KEY = "principal"; + + private static final String KEY_TAB_FILE_PATH_KEY = "keyTabFilePath"; + + private static final String KRB_REALM_KEY = "krbRealm"; + + private static final String KRB_HOST_FQDN_KEY = "krbHostFQDN"; + + private static final String KRB_SERVICE_NAME_KEY = "krbServiceName"; + + private static final String ENABLE_PARTITION_KEY = "enablePartition"; + + private static final String PARTITION_FIELDS_KEY = "partitionFields"; + + private static final String URL_KEY = "url"; + + private static final String TABLE_NAME_KEY = "tableName"; + + private static final String USER_NAME_KEY = "userName"; + + private static final String PASSWORD_KEY = "password"; + + private static final String BATCH_SIZE_KEY = "batchSize"; + + private static final String BATCH_WAIT_INTERVAL_KEY = "batchWaitInterval"; + + private static final String BUFFER_SIZE_KEY = "bufferSize"; + + private static final String FLUSH_INTERVAL_MS_KEY = "flushIntervalMs"; + + private static final String SCHEMA_KEY = "schema"; + + private static final String UPDATE_KEY = "updateMode"; + + private static final String KUDU_TYPE = "kudu"; + + private static final String STORE_TYPE_KEY = "storeType"; + + private static final String KRB_DEFAULT_REALM = "HADOOP.COM"; + + private static final String CURRENT_TYPE = "impala"; - private static final String CURR_TYPE = "impala"; @Override - public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { + public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { ImpalaTableInfo impalaTableInfo = new ImpalaTableInfo(); impalaTableInfo.setName(tableName); parseFieldsInfo(fieldsInfo, impalaTableInfo); - impalaTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(ImpalaTableInfo.PARALLELISM_KEY.toLowerCase()))); - impalaTableInfo.setUrl(MathUtil.getString(props.get(ImpalaTableInfo.URL_KEY.toLowerCase()))); - impalaTableInfo.setTableName(MathUtil.getString(props.get(ImpalaTableInfo.TABLE_NAME_KEY.toLowerCase()))); - impalaTableInfo.setBatchSize(MathUtil.getIntegerVal(props.get(ImpalaTableInfo.BATCH_SIZE_KEY.toLowerCase()))); - impalaTableInfo.setBatchWaitInterval(MathUtil.getLongVal(props.get(ImpalaTableInfo.BATCH_WAIT_INTERVAL_KEY.toLowerCase()))); - impalaTableInfo.setBufferSize(MathUtil.getString(props.get(ImpalaTableInfo.BUFFER_SIZE_KEY.toLowerCase()))); - impalaTableInfo.setFlushIntervalMs(MathUtil.getString(props.get(ImpalaTableInfo.FLUSH_INTERVALMS_KEY.toLowerCase()))); - impalaTableInfo.setSchema(MathUtil.getString(props.get(ImpalaTableInfo.SCHEMA_KEY.toLowerCase()))); - impalaTableInfo.setUpdateMode(MathUtil.getString(props.get(ImpalaTableInfo.UPDATE_KEY.toLowerCase()))); - - Integer authMech = MathUtil.getIntegerVal(props.get(ImpalaTableInfo.AUTHMECH_KEY.toLowerCase())); + impalaTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PARALLELISM_KEY.toLowerCase()))); + impalaTableInfo.setUrl(MathUtil.getString(props.get(URL_KEY.toLowerCase()))); + impalaTableInfo.setTableName(MathUtil.getString(props.get(TABLE_NAME_KEY.toLowerCase()))); + impalaTableInfo.setBatchSize(MathUtil.getIntegerVal(props.get(BATCH_SIZE_KEY.toLowerCase()))); + impalaTableInfo.setBatchWaitInterval(MathUtil.getLongVal(props.get(BATCH_WAIT_INTERVAL_KEY.toLowerCase()))); + impalaTableInfo.setBufferSize(MathUtil.getString(props.get(BUFFER_SIZE_KEY.toLowerCase()))); + impalaTableInfo.setFlushIntervalMs(MathUtil.getString(props.get(FLUSH_INTERVAL_MS_KEY.toLowerCase()))); + impalaTableInfo.setSchema(MathUtil.getString(props.get(SCHEMA_KEY.toLowerCase()))); + impalaTableInfo.setUpdateMode(MathUtil.getString(props.get(UPDATE_KEY.toLowerCase()))); + + Integer authMech = MathUtil.getIntegerVal(props.get(AUTH_MECH_KEY.toLowerCase())); authMech = authMech == null ? 0 : authMech; impalaTableInfo.setAuthMech(authMech); - List authMechs = Arrays.asList(new Integer[]{0, 1, 2, 3}); + List authMechs = Arrays.asList(0, 1, 2, 3); if (!authMechs.contains(authMech)) { throw new IllegalArgumentException("The value of authMech is illegal, Please select 0, 1, 2, 3"); } else if (authMech == 1) { - impalaTableInfo.setPrincipal(MathUtil.getString(props.get(ImpalaTableInfo.PRINCIPAL_KEY.toLowerCase()))); - impalaTableInfo.setKeyTabFilePath(MathUtil.getString(props.get(ImpalaTableInfo.KEYTABFILEPATH_KEY.toLowerCase()))); - impalaTableInfo.setKrb5FilePath(MathUtil.getString(props.get(ImpalaTableInfo.KRB5FILEPATH_KEY.toLowerCase()))); - String krbRealm = MathUtil.getString(props.get(ImpalaTableInfo.KRBREALM_KEY.toLowerCase())); - krbRealm = krbRealm == null ? "HADOOP.COM" : krbRealm; + impalaTableInfo.setPrincipal(MathUtil.getString(props.get(PRINCIPAL_KEY.toLowerCase()))); + impalaTableInfo.setKeyTabFilePath(MathUtil.getString(props.get(KEY_TAB_FILE_PATH_KEY.toLowerCase()))); + impalaTableInfo.setKrb5FilePath(MathUtil.getString(props.get(KRB5FILEPATH_KEY.toLowerCase()))); + String krbRealm = MathUtil.getString(props.get(KRB_REALM_KEY.toLowerCase())); + krbRealm = krbRealm == null ? KRB_DEFAULT_REALM : krbRealm; impalaTableInfo.setKrbRealm(krbRealm); - impalaTableInfo.setKrbHostFQDN(MathUtil.getString(props.get(ImpalaTableInfo.KRBHOSTFQDN_KEY.toLowerCase()))); - impalaTableInfo.setKrbServiceName(MathUtil.getString(props.get(ImpalaTableInfo.KRBSERVICENAME_KEY.toLowerCase()))); + impalaTableInfo.setKrbHostFQDN(MathUtil.getString(props.get(KRB_HOST_FQDN_KEY.toLowerCase()))); + impalaTableInfo.setKrbServiceName(MathUtil.getString(props.get(KRB_SERVICE_NAME_KEY.toLowerCase()))); } else if (authMech == 2) { - impalaTableInfo.setUserName(MathUtil.getString(props.get(ImpalaTableInfo.USER_NAME_KEY.toLowerCase()))); + impalaTableInfo.setUserName(MathUtil.getString(props.get(USER_NAME_KEY.toLowerCase()))); } else if (authMech == 3) { - impalaTableInfo.setUserName(MathUtil.getString(props.get(ImpalaTableInfo.USER_NAME_KEY.toLowerCase()))); - impalaTableInfo.setPassword(MathUtil.getString(props.get(ImpalaTableInfo.PASSWORD_KEY.toLowerCase()))); + impalaTableInfo.setUserName(MathUtil.getString(props.get(USER_NAME_KEY.toLowerCase()))); + impalaTableInfo.setPassword(MathUtil.getString(props.get(PASSWORD_KEY.toLowerCase()))); } - String enablePartitionStr = (String) props.get(ImpalaTableInfo.ENABLEPARITION_KEY.toLowerCase()); + String storeType = MathUtil.getString(props.get(STORE_TYPE_KEY.toLowerCase())); + impalaTableInfo.setStoreType(storeType); + + String enablePartitionStr = (String) props.get(ENABLE_PARTITION_KEY.toLowerCase()); boolean enablePartition = MathUtil.getBoolean(enablePartitionStr == null ? "false" : enablePartitionStr); impalaTableInfo.setEnablePartition(enablePartition); - if (enablePartition) { - String partitionFields = MathUtil.getString(props.get(ImpalaTableInfo.PARTITIONFIELDS_KEY.toLowerCase())); + + if (!storeType.equalsIgnoreCase(KUDU_TYPE) && enablePartition) { + String partitionFields = MathUtil.getString(props.get(PARTITION_FIELDS_KEY.toLowerCase())); impalaTableInfo.setPartitionFields(partitionFields); } + impalaTableInfo.setType(CURRENT_TYPE); + impalaTableInfo.check(); return impalaTableInfo; } - - @Override - public Class dbTypeConvertToJavaType(String fieldType) { - switch (fieldType.toLowerCase()) { - case "boolean": - return Boolean.class; - case "char": - return Character.class; - case "double": - return Double.class; - case "float": - return Float.class; - case "tinyint": - return Byte.class; - case "smallint": - return Short.class; - case "int": - return Integer.class; - case "bigint": - return Long.class; - case "decimal": - return BigDecimal.class; - case "string": - case "varchar": - return String.class; - case "timestamp": - return Timestamp.class; - default: - break; - } - - throw new RuntimeException("不支持 " + fieldType + " 类型"); - } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java index 032d9ea18..bc0dbac23 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaTableInfo.java @@ -16,43 +16,46 @@ * limitations under the License. */ - package com.dtstack.flink.sql.sink.impala.table; -import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; +import com.dtstack.flink.sql.enums.EUpdateMode; +import com.dtstack.flink.sql.table.AbstractTargetTableInfo; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import java.util.Objects; + /** - * Date: 2019/11/13 + * Date: 2020/10/14 * Company: www.dtstack.com * - * @author xiuzhu + * @author tiezhu */ +public class ImpalaTableInfo extends AbstractTargetTableInfo { -public class ImpalaTableInfo extends RdbTableInfo { + private static final int MAX_BATCH_SIZE = 10000; - public static final String AUTHMECH_KEY = "authMech"; + private String url; - public static final String KRB5FILEPATH_KEY = "krb5FilePath"; + private String tableName; - public static final String PRINCIPAL_KEY = "principal"; + private String userName; - public static final String KEYTABFILEPATH_KEY = "keyTabFilePath"; + private String password; - public static final String KRBREALM_KEY = "krbRealm"; + private Integer batchSize; - public static final String KRBHOSTFQDN_KEY = "krbHostFQDN"; + private Long batchWaitInterval; - public static final String KRBSERVICENAME_KEY = "krbServiceName"; + private String bufferSize; - public static final String ENABLEPARITION_KEY = "enablePartition"; + private String flushIntervalMs; - public static final String PARTITIONFIELDS_KEY = "partitionFields"; + private String schema; - private static final String CURR_TYPE = "impala"; + private boolean allReplace; - private static final String PARTITION_FIELD_SPLIT_REGEX = ","; + private String updateMode; private Integer authMech; @@ -68,12 +71,90 @@ public class ImpalaTableInfo extends RdbTableInfo { private String krbServiceName; - private boolean enablePartition; + private boolean enablePartition = false; + + private String partitionFields; + + private String storeType; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + + public Long getBatchWaitInterval() { + return batchWaitInterval; + } + + public void setBatchWaitInterval(Long batchWaitInterval) { + this.batchWaitInterval = batchWaitInterval; + } + + public String getBufferSize() { + return bufferSize; + } + + public void setBufferSize(String bufferSize) { + this.bufferSize = bufferSize; + } + + public String getFlushIntervalMs() { + return flushIntervalMs; + } + + public void setFlushIntervalMs(String flushIntervalMs) { + this.flushIntervalMs = flushIntervalMs; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } - private String[] partitionFields; + public String getUpdateMode() { + return updateMode; + } - public ImpalaTableInfo() { - setType(CURR_TYPE); + public void setUpdateMode(String updateMode) { + this.updateMode = updateMode; } public Integer getAuthMech() { @@ -140,38 +221,49 @@ public void setEnablePartition(boolean enablePartition) { this.enablePartition = enablePartition; } - public String[] getPartitionFields() { + public String getPartitionFields() { return partitionFields; } public void setPartitionFields(String partitionFields) { - this.partitionFields = StringUtils.split(partitionFields, PARTITION_FIELD_SPLIT_REGEX); + this.partitionFields = partitionFields; + } + + public String getStoreType() { + return storeType; + } + + public void setStoreType(String storeType) { + this.storeType = storeType; + } + + @Override + public String getType() { + return super.getType().toLowerCase(); } @Override public boolean check() { - Preconditions.checkNotNull(this.getUrl(), "impala field of url is required"); - Preconditions.checkNotNull(this.getTableName(), "impala field of tableName is required"); - Preconditions.checkNotNull(this.getAuthMech(), "impala field of authMech is required"); - Integer authMech = getAuthMech(); - - if (authMech == 1) { - Preconditions.checkNotNull(this.getKrb5FilePath(), "impala field of krb5FilePath is required"); - Preconditions.checkNotNull(this.getPrincipal(), "impala field of principal is required"); - Preconditions.checkNotNull(this.getKeyTabFilePath(), "impala field of keyTabFilePath is required"); - Preconditions.checkNotNull(this.getKrbHostFQDN(), "impala field of krbHostFQDN is required"); - Preconditions.checkNotNull(this.getKrbServiceName(), "impala field of krbServiceName is required"); - } else if (authMech == 2) { - Preconditions.checkNotNull(this.getUserName(), "impala field of userName is required"); - }else if (authMech == 3) { - Preconditions.checkNotNull(this.getUserName(), "impala field of userName is required"); - Preconditions.checkNotNull(this.getPassword(), "impala field of password is required"); + Preconditions.checkNotNull(url, "impala field of URL is required"); + Preconditions.checkNotNull(tableName, "impala field of tableName is required"); + + if (Objects.nonNull(batchSize)) { + Preconditions.checkArgument(batchSize <= MAX_BATCH_SIZE, "batchSize must be less than " + MAX_BATCH_SIZE); } - if (isEnablePartition()) { - Preconditions.checkArgument(this.getPartitionFields().length > 0, "impala field of partitionFields is required"); + if (StringUtils.equalsIgnoreCase(updateMode, EUpdateMode.UPSERT.name())) { + Preconditions.checkArgument(Objects.nonNull(getPrimaryKeys()) && getPrimaryKeys().size() > 0, "updateMode mode primary is required"); } + if (Objects.nonNull(getPrimaryKeys())) { + getPrimaryKeys().forEach(pk -> { + Preconditions.checkArgument(getFieldList().contains(pk), "primary key " + pk + " not found in sink table field"); + }); + } + + + Preconditions.checkArgument(getFieldList().size() == getFieldExtraInfoList().size(), + "fields and fieldExtraInfoList attributes must be the same length"); return true; } } diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index cf7f0dea7..0b30729f8 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -7,6 +7,7 @@ import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -18,6 +19,7 @@ import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; @@ -31,6 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.SQLException; import java.sql.Timestamp; import java.util.Arrays; @@ -218,24 +222,8 @@ private String buildKey(Map val, List equalFieldList) { private KuduScanner getConn(KuduSideTableInfo tableInfo) { try { if (client == null) { - String kuduMasters = tableInfo.getKuduMasters(); String tableName = tableInfo.getTableName(); - Integer workerCount = tableInfo.getWorkerCount(); - Integer defaultSocketReadTimeoutMs = tableInfo.getDefaultSocketReadTimeoutMs(); - Integer defaultOperationTimeoutMs = tableInfo.getDefaultOperationTimeoutMs(); - - Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); - - KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(kuduMasters); - if (null != workerCount) { - kuduClientBuilder.workerCount(workerCount); - } - - if (null != defaultOperationTimeoutMs) { - kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); - } - client = kuduClientBuilder.build(); - + client = getClient(tableInfo); if (!client.tableExists(tableName)) { throw new IllegalArgumentException("Table Open Failed , please check table exists"); } @@ -250,6 +238,36 @@ private KuduScanner getConn(KuduSideTableInfo tableInfo) { } } + private KuduClient getClient(KuduSideTableInfo tableInfo) throws IOException { + String kuduMasters = tableInfo.getKuduMasters(); + Integer workerCount = tableInfo.getWorkerCount(); + Integer defaultOperationTimeoutMs = tableInfo.getDefaultOperationTimeoutMs(); + + Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); + + KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(kuduMasters); + + if (null != workerCount) { + kuduClientBuilder.workerCount(workerCount); + } + + if (null != defaultOperationTimeoutMs) { + kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); + } + + if (tableInfo.isEnableKrb()) { + UserGroupInformation ugi = KrbUtils.getUgi(tableInfo.getPrincipal(), tableInfo.getKeytab(), tableInfo.getKrb5conf()); + return ugi.doAs(new PrivilegedAction() { + @Override + public KuduClient run() { + return kuduClientBuilder.build(); + } + }); + } else { + return kuduClientBuilder.build(); + } + } + /** * @param builder 创建AsyncKuduScanner对象 diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index edb49814f..43fda6e9f 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -10,6 +10,7 @@ import com.dtstack.flink.sql.side.cache.CacheObj; import com.dtstack.flink.sql.side.kudu.table.KuduSideTableInfo; import com.dtstack.flink.sql.side.kudu.utils.KuduUtil; +import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.stumbleupon.async.Callback; @@ -23,6 +24,7 @@ import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.AsyncKuduClient; @@ -35,6 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.security.PrivilegedAction; import java.sql.Timestamp; import java.util.*; @@ -80,25 +84,10 @@ public void open(Configuration parameters) throws Exception { * * @throws KuduException */ - private void connKuDu() throws KuduException { + private void connKuDu() throws IOException { if (null == table) { - String kuduMasters = kuduSideTableInfo.getKuduMasters(); String tableName = kuduSideTableInfo.getTableName(); - Integer workerCount = kuduSideTableInfo.getWorkerCount(); - Integer defaultSocketReadTimeoutMs = kuduSideTableInfo.getDefaultSocketReadTimeoutMs(); - Integer defaultOperationTimeoutMs = kuduSideTableInfo.getDefaultOperationTimeoutMs(); - - Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); - - AsyncKuduClient.AsyncKuduClientBuilder asyncKuduClientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasters); - if (null != workerCount) { - asyncKuduClientBuilder.workerCount(workerCount); - } - - if (null != defaultOperationTimeoutMs) { - asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); - } - asyncClient = asyncKuduClientBuilder.build(); + asyncClient = getClient(); if (!asyncClient.syncClient().tableExists(tableName)) { throw new IllegalArgumentException("Table Open Failed , please check table exists"); } @@ -128,6 +117,40 @@ private void connKuDu() throws KuduException { scannerBuilder.setProjectedColumnNames(projectColumns); } + private AsyncKuduClient getClient() throws IOException { + String kuduMasters = kuduSideTableInfo.getKuduMasters(); + Integer workerCount = kuduSideTableInfo.getWorkerCount(); + Integer defaultOperationTimeoutMs = kuduSideTableInfo.getDefaultOperationTimeoutMs(); + + Preconditions.checkNotNull(kuduMasters, "kuduMasters could not be null"); + + AsyncKuduClient.AsyncKuduClientBuilder asyncKuduClientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasters); + if (null != workerCount) { + asyncKuduClientBuilder.workerCount(workerCount); + } + + if (null != defaultOperationTimeoutMs) { + asyncKuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); + } + + if (kuduSideTableInfo.isEnableKrb()) { + UserGroupInformation ugi = KrbUtils.getUgi( + kuduSideTableInfo.getPrincipal(), + kuduSideTableInfo.getKeytab(), + kuduSideTableInfo.getKrb5conf() + ); + return ugi.doAs( + new PrivilegedAction() { + @Override + public AsyncKuduClient run() { + return asyncKuduClientBuilder.build(); + } + }); + } else { + return asyncKuduClientBuilder.build(); + } + } + @Override public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { CRow inputCopy = new CRow(input.row(), input.change()); diff --git a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java index 62d215d87..f350dd6c3 100644 --- a/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java +++ b/kudu/kudu-side/kudu-side-core/src/main/java/com/dtstack/flink/sql/side/kudu/table/KuduSideParser.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.side.kudu.table; +import com.dtstack.flink.sql.constrant.PluginParamConsts; import com.dtstack.flink.sql.table.AbstractSideTableParser; import com.dtstack.flink.sql.table.AbstractTableInfo; import com.dtstack.flink.sql.util.MathUtil; @@ -71,6 +72,18 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map() { + @Override + public AsyncKuduClient run() { + return asyncKuduClientBuilder.build(); + } + }); + } else { + client = asyncKuduClientBuilder.build(); + } + LOG.info("connect kudu is successed!"); + KuduClient syncClient = client.syncClient(); if (syncClient.tableExists(tableName)) { table = syncClient.openTable(tableName); } @@ -215,6 +240,25 @@ public KuduOutputFormatBuilder setDefaultSocketReadTimeoutMs(Integer defaultSock return this; } + public KuduOutputFormatBuilder setPrincipal(String principal) { + kuduOutputFormat.principal = principal; + return this; + } + + public KuduOutputFormatBuilder setKeytab(String keytab) { + kuduOutputFormat.keytab = keytab; + return this; + } + + public KuduOutputFormatBuilder setKrb5conf(String krb5conf) { + kuduOutputFormat.krb5conf = krb5conf; + return this; + } + + public KuduOutputFormatBuilder setEnableKrb(boolean enableKrb) { + kuduOutputFormat.enableKrb = enableKrb; + return this; + } public KuduOutputFormat finish() { if (kuduOutputFormat.kuduMasters == null) { diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index 4c7c3bea8..f8ac125be 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -38,6 +38,11 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStr private int parallelism = -1; + private String principal; + private String keytab; + private String krb5conf; + boolean enableKrb; + @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { KuduTableInfo kuduTableInfo = (KuduTableInfo) targetTableInfo; @@ -47,12 +52,20 @@ public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { this.defaultSocketReadTimeoutMs = kuduTableInfo.getDefaultSocketReadTimeoutMs(); this.workerCount = kuduTableInfo.getWorkerCount(); this.writeMode = kuduTableInfo.getWriteMode(); + this.principal = kuduTableInfo.getPrincipal(); + this.keytab = kuduTableInfo.getKeytab(); + this.krb5conf = kuduTableInfo.getKrb5conf(); + this.enableKrb = kuduTableInfo.isEnableKrb(); return this; } @Override public void emitDataStream(DataStream> dataStream) { + consumeDataStream(dataStream); + } + + public DataStreamSink> consumeDataStream(DataStream> dataStream) { KuduOutputFormat.KuduOutputFormatBuilder builder = KuduOutputFormat.buildKuduOutputFormat(); builder.setKuduMasters(this.kuduMasters) .setTableName(this.tableName) @@ -61,7 +74,11 @@ public void emitDataStream(DataStream> dataStream) { .setDefaultOperationTimeoutMs(this.defaultOperationTimeoutMs) .setDefaultSocketReadTimeoutMs(this.defaultSocketReadTimeoutMs) .setFieldNames(this.fieldNames) - .setFieldTypes(this.fieldTypes); + .setFieldTypes(this.fieldTypes) + .setPrincipal(this.principal) + .setKeytab(this.keytab) + .setKrb5conf(this.krb5conf) + .setEnableKrb(this.enableKrb); KuduOutputFormat kuduOutputFormat = builder.finish(); RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(kuduOutputFormat); DataStreamSink dataStreamSink = dataStream.addSink(richSinkFunction); @@ -69,6 +86,7 @@ public void emitDataStream(DataStream> dataStream) { if (parallelism > 0) { dataStreamSink.setParallelism(parallelism); } + return dataStreamSink; } @Override diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java index 20302d44f..80e9cc628 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java @@ -1,5 +1,6 @@ package com.dtstack.flink.sql.sink.kudu.table; +import com.dtstack.flink.sql.constrant.PluginParamConsts; import com.dtstack.flink.sql.sink.kudu.KuduOutputFormat; import com.dtstack.flink.sql.table.AbstractTableParser; import com.dtstack.flink.sql.table.AbstractTableInfo; @@ -39,6 +40,18 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map resultFuture){ + protected void preInvoke(CRow input, ResultFuture resultFuture) { } @@ -120,8 +123,8 @@ protected void preInvoke(CRow input, ResultFuture resultFuture){ public void handleAsyncInvoke(Map inputParams, CRow input, ResultFuture resultFuture) throws Exception { AtomicLong networkLogCounter = new AtomicLong(0L); - while (!connectionStatus.get()){//network is unhealth - if(networkLogCounter.getAndIncrement() % 1000 == 0){ + while (!connectionStatus.get()) {//network is unhealth + if (networkLogCounter.getAndIncrement() % 1000 == 0) { LOG.info("network unhealth to block task"); } Thread.sleep(100); @@ -133,53 +136,84 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul private void connectWithRetry(Map inputParams, CRow input, ResultFuture resultFuture, SQLClient rdbSqlClient) { AtomicLong failCounter = new AtomicLong(0); AtomicBoolean finishFlag = new AtomicBoolean(false); - while(!finishFlag.get()){ - try{ + while (!finishFlag.get()) { + try { CountDownLatch latch = new CountDownLatch(1); - rdbSqlClient.getConnection(conn -> { - try { - if(conn.failed()){ - connectionStatus.set(false); - if(failCounter.getAndIncrement() % 1000 == 0){ - LOG.error("getConnection error", conn.cause()); - } - if(failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)){ - resultFuture.completeExceptionally(conn.cause()); - finishFlag.set(true); - } - return; - } - connectionStatus.set(true); - ScheduledFuture timerFuture = registerTimer(input, resultFuture); - cancelTimerWhenComplete(resultFuture, timerFuture); - handleQuery(conn.result(), inputParams, input, resultFuture); - finishFlag.set(true); - } catch (Exception e) { - dealFillDataError(input, resultFuture, e); - } finally { - latch.countDown(); - } - }); + asyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); try { latch.await(); } catch (InterruptedException e) { LOG.error("", e); } - } catch (Exception e){ + } catch (Exception e) { //数据源队列溢出情况 connectionStatus.set(false); } - if(!finishFlag.get()){ + if (!finishFlag.get()) { try { Thread.sleep(3000); - } catch (Exception e){ + } catch (Exception e) { LOG.error("", e); } } } } + protected void asyncQueryData(Map inputParams, + CRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { + doAsyncQueryData(inputParams, + input, resultFuture, + rdbSqlClient, + failCounter, + finishFlag, + latch); + } + + final protected void doAsyncQueryData( + Map inputParams, + CRow input, + ResultFuture resultFuture, + SQLClient rdbSqlClient, + AtomicLong failCounter, + AtomicBoolean finishFlag, + CountDownLatch latch) { + rdbSqlClient.getConnection(conn -> { + try { + if (conn.failed()) { + connectionStatus.set(false); + if (failCounter.getAndIncrement() % 1000 == 0) { + LOG.error("getConnection error", conn.cause()); + } + if (failCounter.get() >= sideInfo.getSideTableInfo().getConnectRetryMaxNum(100)) { + resultFuture.completeExceptionally(conn.cause()); + finishFlag.set(true); + } + return; + } + connectionStatus.set(true); + preInvoke(input, resultFuture); + + handleQuery(conn.result(), inputParams, input, resultFuture); + finishFlag.set(true); + } catch (Exception e) { + dealFillDataError(input, resultFuture, e); + } finally { + latch.countDown(); + } + }); + } + private Object convertDataType(Object val) { if (val == null) { @@ -219,7 +253,7 @@ private Object convertDataType(Object val) { @Override public String buildCacheKey(Map inputParam) { - return StringUtils.join(inputParam.values(),"_"); + return StringUtils.join(inputParam.values(), "_"); } @Override @@ -231,7 +265,7 @@ public Row fillData(Row input, Object line) { boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(sideInfo.getRowTypeInfo().getTypeAt(entry.getValue()).getClass()); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 - obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); + obj = ((Timestamp) obj).getTime() + (long) LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } row.setField(entry.getKey(), obj); @@ -258,7 +292,7 @@ public void close() throws Exception { rdbSqlClient.close(); } - if(executor != null){ + if (executor != null) { executor.shutdown(); } @@ -268,7 +302,7 @@ public void setRdbSqlClient(SQLClient rdbSqlClient) { this.rdbSqlClient = rdbSqlClient; } - private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture){ + private void handleQuery(SQLConnection connection, Map inputParams, CRow input, ResultFuture resultFuture) { String key = buildCacheKey(inputParams); JsonArray params = new JsonArray(Lists.newArrayList(inputParams.values())); connection.queryWithParams(sideInfo.getSqlCondition(), params, rs -> { @@ -312,9 +346,9 @@ private void handleQuery(SQLConnection connection, Map inputPara }); } - private Map formatInputParam(Map inputParam){ + private Map formatInputParam(Map inputParam) { Map result = Maps.newHashMap(); - inputParam.forEach((k,v) -> { + inputParam.forEach((k, v) -> { result.put(k, convertDataType(v)); }); return result; diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java index b1a31de42..f06ecba74 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/JDBCTypeConvertUtils.java @@ -228,4 +228,59 @@ public static int[] buildSqlTypes(List fieldTypeArray) { return tmpFieldsType; } + /** + * according to Java type, get the corresponding SQL type + * + * @param fieldTypeList the Java type + * @return the type number of the corresponding type + */ + public static int[] getSqlTypeFromFieldType(List fieldTypeList) { + int[] tmpFieldsType = new int[fieldTypeList.size()]; + for (int i = 0; i < fieldTypeList.size(); i++) { + String fieldType = fieldTypeList.get(i).toUpperCase(); + switch (fieldType) { + case "INT": + tmpFieldsType[i] = Types.INTEGER; + break; + case "BOOLEAN": + tmpFieldsType[i] = Types.BOOLEAN; + break; + case "BIGINT": + tmpFieldsType[i] = Types.BIGINT; + break; + case "SHORT": + tmpFieldsType[i] = Types.SMALLINT; + break; + case "STRING": + case "CHAR": + tmpFieldsType[i] = Types.CHAR; + break; + case "BYTE": + tmpFieldsType[i] = Types.BINARY; + break; + case "FLOAT": + tmpFieldsType[i] = Types.FLOAT; + break; + case "DOUBLE": + tmpFieldsType[i] = Types.DOUBLE; + break; + case "TIMESTAMP": + tmpFieldsType[i] = Types.TIMESTAMP; + break; + case "BIGDECIMAL": + tmpFieldsType[i] = Types.DECIMAL; + break; + case "DATE": + tmpFieldsType[i] = Types.DATE; + break; + case "TIME": + tmpFieldsType[i] = Types.TIME; + break; + default: + throw new RuntimeException("no support field type for sql. the input type:" + fieldType); + } + } + return tmpFieldsType; + } + } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java index b87c2e042..c2491e586 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/JDBCUpsertOutputFormat.java @@ -29,19 +29,18 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.types.Row; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.sql.ResultSet; +import java.security.PrivilegedAction; import java.sql.SQLException; -import java.sql.Statement; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -77,7 +76,6 @@ public class JDBCUpsertOutputFormat extends AbstractJDBCOutputFormat tuple2) throws IOException { - if(!flushFlag.get()){ - throw new RuntimeException("connect exception,can not write record"); - } checkConnectionOpen(); try { if (outRecords.getCount() % RECEIVEDATA_PRINT_FREQUENTY == 0 || LOG.isDebugEnabled()) { @@ -193,7 +190,6 @@ private void checkConnectionOpen() { public synchronized void flush() throws Exception { jdbcWriter.executeBatch(connection); batchCount = 0; - flushFlag.set(true); } /** From 92dd4327c67f78c9625a2629e077e8536955d0f3 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 27 Oct 2020 09:35:07 +0800 Subject: [PATCH 332/378] [fix] impala varchar data error; storeType npe; value condition bug --- .../sql/sink/impala/ImpalaOutputFormat.java | 17 +++++++++-------- .../sql/sink/impala/table/ImpalaSinkParser.java | 7 +++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 19e904ef2..e880f58d4 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -76,7 +76,7 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted - private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp"}; + private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; private static final Integer DEFAULT_CONN_TIME_OUT = 60; private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; @@ -327,10 +327,7 @@ private void putRowIntoMap(Map> rowDataMap, Tuple2 rebuildFieldNameListAndTypeList(List fieldNames, - List staticPartitionFields, - List fieldTypes, - String partitionFields) { + private List rebuildFieldNameListAndTypeList(List fieldNames, List staticPartitionFields, List fieldTypes, String partitionFields) { if (partitionFields.isEmpty()) { return fieldNames; } @@ -364,8 +361,8 @@ private List rebuildFieldNameListAndTypeList(List fieldNames, * @return quoted condition */ private String valueConditionAddQuotation(String valueCondition) { - final String[] valueConditionCopy = {valueCondition}; String[] temps = valueCondition.split(","); + List replacedItem = new ArrayList<>(); Arrays.stream(temps).forEach( item -> { Matcher matcher = TYPE_PATTERN.matcher(item); @@ -375,13 +372,15 @@ private String valueConditionAddQuotation(String valueCondition) { if (Arrays.asList(NEED_QUOTE_TYPE).contains(type)) { if (!"null".equals(value)) { - valueConditionCopy[0] = valueConditionCopy[0].replace(value, "'" + value + "'"); + item = item.replace(value, "'" + value + "'"); } } } + replacedItem.add(item); } ); - return "(" + valueConditionCopy[0] + ")"; + + return "(" + String.join(", ", replacedItem) + ")"; } @Override @@ -746,6 +745,8 @@ public ImpalaOutputFormat build() { checkNotNull(format.password, "password is required!"); } + checkNotNull(format.storeType, "storeType is required!"); + return format; } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java index f1a475f61..003b5a6ab 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/table/ImpalaSinkParser.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Date: 2020/10/14 @@ -76,6 +77,8 @@ public class ImpalaSinkParser extends AbstractTableParser { private static final String KUDU_TYPE = "kudu"; + private static final String DEFAULT_STORE_TYPE = "kudu"; + private static final String STORE_TYPE_KEY = "storeType"; private static final String KRB_DEFAULT_REALM = "HADOOP.COM"; @@ -123,13 +126,13 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map Date: Wed, 28 Oct 2020 14:24:24 +0800 Subject: [PATCH 333/378] [fix] varchar type bug --- .../flink/sql/sink/impala/ImpalaOutputFormat.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index e880f58d4..ed3370727 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -52,6 +52,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -370,9 +371,11 @@ private String valueConditionAddQuotation(String valueCondition) { String value = matcher.group(1); String type = matcher.group(2); - if (Arrays.asList(NEED_QUOTE_TYPE).contains(type)) { - if (!"null".equals(value)) { - item = item.replace(value, "'" + value + "'"); + for (String needQuoteType : NEED_QUOTE_TYPE) { + if (type.contains(needQuoteType)) { + if (!"null".equals(value)) { + item = item.replace(value, "'" + value + "'"); + } } } } @@ -580,8 +583,10 @@ private String buildTableFieldsCondition(List fieldNames, String partiti private String buildValuesCondition(List fieldTypes, Row row) { String valuesCondition = fieldTypes.stream().map( f -> { - if (Arrays.asList(NEED_QUOTE_TYPE).contains(f.toLowerCase())) { - return String.format("cast(? as %s)", f.toLowerCase()); + for(String item : NEED_QUOTE_TYPE) { + if (f.toLowerCase().contains(item)) { + return String.format("cast(? as %s)", f.toLowerCase()); + } } return "?"; }).collect(Collectors.joining(", ")); From c0b80769aa5b162b23e1cdb84280170cd22297ba Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 28 Oct 2020 15:08:06 +0800 Subject: [PATCH 334/378] [fix] fix krb bug --- .../java/com/dtstack/flink/sql/util/KrbUtils.java | 13 +++++++++++-- .../com/dtstack/flink/sql/util/KrbUtilsTest.java | 2 +- .../flink/sql/side/impala/ImpalaAllReqRow.java | 2 +- .../flink/sql/side/impala/ImpalaAsyncReqRow.java | 3 +-- .../flink/sql/sink/impala/ImpalaOutputFormat.java | 2 +- .../dtstack/flink/sql/side/kudu/KuduAllReqRow.java | 2 +- .../flink/sql/side/kudu/KuduAsyncReqRow.java | 3 +-- .../flink/sql/sink/kudu/KuduOutputFormat.java | 2 +- 8 files changed, 18 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java index 2a83359c4..00f2f538d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/KrbUtils.java @@ -20,8 +20,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.security.krb5.Config; +import sun.security.krb5.KrbException; import java.io.IOException; @@ -40,9 +43,15 @@ public class KrbUtils { // public static final String FALSE_STR = "false"; // public static final String SUBJECT_ONLY_KEY = "javax.security.auth.useSubjectCredsOnly"; - public static UserGroupInformation getUgi(String principal, String keytabPath, String krb5confPath) throws IOException { - LOG.info("Kerberos login with principal: {} and keytab: {}", principal, keytabPath); + public static UserGroupInformation loginAndReturnUgi(String principal, String keytabPath, String krb5confPath) throws IOException { LOG.info("Kerberos login with principal: {} and keytab: {}", principal, keytabPath); System.setProperty(KRB5_CONF_KEY, krb5confPath); + // 不刷新会读/etc/krb5.conf + try { + Config.refresh(); + KerberosName.resetDefaultRealm(); + } catch (KrbException e) { + LOG.warn("resetting default realm failed, current default realm will still be used.", e); + } // TODO 尚未探索出此选项的意义，以后研究明白方可打开 // System.setProperty(SUBJECT_ONLY_KEY, FALSE_STR); Configuration configuration = new Configuration(); diff --git a/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java index 6c7359e14..5d65158c7 100644 --- a/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/util/KrbUtilsTest.java @@ -35,7 +35,7 @@ public void testGetUgi() throws IOException { String keytabPath = ""; String krb5confPath = ""; try { - KrbUtils.getUgi(principal, keytabPath, krb5confPath); + KrbUtils.loginAndReturnUgi(principal, keytabPath, krb5confPath); } catch (IllegalArgumentException e) { Assert.assertEquals(e.getMessage(), "Can't get Kerberos realm"); } diff --git a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java index 179e52e56..85b1f1b1f 100644 --- a/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java +++ b/impala/impala-side/impala-all-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAllReqRow.java @@ -71,7 +71,7 @@ public Connection getConn(String dbUrl, String userName, String password) { String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); String krb5FilePath = impalaSideTableInfo.getKrb5FilePath(); String principal = impalaSideTableInfo.getPrincipal(); - UserGroupInformation ugi = KrbUtils.getUgi(principal, keyTabFilePath, krb5FilePath); + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi(principal, keyTabFilePath, krb5FilePath); connection = ugi.doAs((PrivilegedAction) () -> { try { return DriverManager.getConnection(url); diff --git a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java index ae8ac2be9..3e7da110a 100644 --- a/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java +++ b/impala/impala-side/impala-async-side/src/main/java/com/dtstack/flink/sql/side/impala/ImpalaAsyncReqRow.java @@ -33,7 +33,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +70,7 @@ public void open(Configuration parameters) throws Exception { String keyTabFilePath = impalaSideTableInfo.getKeyTabFilePath(); String krb5FilePath = impalaSideTableInfo.getKrb5FilePath(); String principal = impalaSideTableInfo.getPrincipal(); - ugi = KrbUtils.getUgi(principal, keyTabFilePath, krb5FilePath); + ugi = KrbUtils.loginAndReturnUgi(principal, keyTabFilePath, krb5FilePath); } openJdbc(parameters); } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index ed3370727..a755ee2bf 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -207,7 +207,7 @@ private void initScheduledTask(Long batchWaitInterval) { private void openConnect() throws IOException { if (authMech == 1) { - UserGroupInformation ugi = KrbUtils.getUgi(principal, keytabPath, krb5confPath); + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi(principal, keytabPath, krb5confPath); try { ugi.doAs((PrivilegedExceptionAction) () -> { openJdbc(); diff --git a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java index 0b30729f8..01ee1cd9b 100644 --- a/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java +++ b/kudu/kudu-side/kudu-all-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAllReqRow.java @@ -256,7 +256,7 @@ private KuduClient getClient(KuduSideTableInfo tableInfo) throws IOException { } if (tableInfo.isEnableKrb()) { - UserGroupInformation ugi = KrbUtils.getUgi(tableInfo.getPrincipal(), tableInfo.getKeytab(), tableInfo.getKrb5conf()); + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi(tableInfo.getPrincipal(), tableInfo.getKeytab(), tableInfo.getKrb5conf()); return ugi.doAs(new PrivilegedAction() { @Override public KuduClient run() { diff --git a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java index 43fda6e9f..84b37ab67 100644 --- a/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java +++ b/kudu/kudu-side/kudu-async-side/src/main/java/com/dtstack/flink/sql/side/kudu/KuduAsyncReqRow.java @@ -15,7 +15,6 @@ import com.google.common.collect.Maps; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; -import io.vertx.core.json.JsonArray; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; @@ -134,7 +133,7 @@ private AsyncKuduClient getClient() throws IOException { } if (kuduSideTableInfo.isEnableKrb()) { - UserGroupInformation ugi = KrbUtils.getUgi( + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi( kuduSideTableInfo.getPrincipal(), kuduSideTableInfo.getKeytab(), kuduSideTableInfo.getKrb5conf() diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index 41047762b..ab6ac13a8 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -116,7 +116,7 @@ private void establishConnection() throws IOException { } if (enableKrb) { - UserGroupInformation ugi = KrbUtils.getUgi( + UserGroupInformation ugi = KrbUtils.loginAndReturnUgi( principal, keytab, krb5conf From 7d943052bd620d9fc4e4d6353f24f67b868d713d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Mon, 2 Nov 2020 19:07:02 +0800 Subject: [PATCH 335/378] =?UTF-8?q?[fix]=20RDB=E5=85=A8=E9=87=8F=E7=BB=B4?= =?UTF-8?q?=E8=A1=A8=E4=BD=BF=E7=94=A8=E5=88=AB=E5=90=8D=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/side/rdb/all/AbstractRdbAllReqRow.java | 10 +++++++--- .../flink/sql/side/rdb/all/RdbAllSideInfo.java | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0530e57e2..0db947d42 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -213,13 +213,17 @@ private void queryAndFillData(Map>> tmpCache, C ResultSet resultSet = statement.executeQuery(sql); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); - String[] fields = sideInfo.getSideTableInfo().getFieldTypes(); + String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + Map sideFieldNamesAndTypes = Maps.newHashMap(); + for (int i = 0; i < sideFieldNames.length; i++) { + sideFieldNamesAndTypes.put(sideFieldNames[i], sideFieldTypes[i]); + } + while (resultSet.next()) { Map oneRow = Maps.newHashMap(); for (String fieldName : sideFieldNames) { Object object = resultSet.getObject(fieldName.trim()); - int fieldIndex = sideInfo.getSideTableInfo().getFieldList().indexOf(fieldName.trim()); - object = SwitchUtil.getTarget(object, fields[fieldIndex]); + object = SwitchUtil.getTarget(object, sideFieldNamesAndTypes.get(fieldName)); oneRow.put(fieldName.trim(), object); } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index 82ec95ca0..cafc715d5 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -36,6 +36,8 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -59,7 +61,18 @@ public RdbAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List selectFields = Lists.newArrayList(); + Map physicalFields = rdbSideTableInfo.getPhysicalFields(); + physicalFields.keySet().forEach( + item -> { + if (Objects.isNull(physicalFields.get(item))) { + selectFields.add(quoteIdentifier(item)); + } else { + selectFields.add(quoteIdentifier(physicalFields.get(item)) + " AS " + quoteIdentifier(item)); + } + } + ); + sqlCondition = getSelectFromStatement(getTableName(rdbSideTableInfo), selectFields, sideTableInfo.getPredicateInfoes()); LOG.info("--------dimension sql query-------\n{}" + sqlCondition); } @@ -68,7 +81,7 @@ public String getAdditionalWhereClause() { } private String getSelectFromStatement(String tableName, List selectFields, List predicateInfoes) { - String fromClause = selectFields.stream().map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String fromClause = String.join(", ", selectFields); String predicateClause = predicateInfoes.stream().map(this::buildFilterCondition).collect(Collectors.joining(" AND ")); String whereClause = buildWhereClause(predicateClause); return "SELECT " + fromClause + " FROM " + tableName + whereClause; From 845b55e7081f64a11974263948293aeebd5afe9e Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 3 Nov 2020 10:44:31 +0800 Subject: [PATCH 336/378] =?UTF-8?q?[fix]=20=E6=94=AF=E6=8C=81=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=88=B3=E8=BD=AC=E5=8C=96=E4=B8=BA=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84Timestamp=E3=80=81Time=E3=80=81Date=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DtNestRowDeserializationSchema.java | 52 +++++++++++++++++-- .../com/dtstack/flink/sql/util/DateUtil.java | 13 +++++ .../com/dtstack/flink/sql/util/MathUtil.java | 16 +++++- .../side/elasticsearch6/util/MathUtil.java | 38 +++++++++----- .../side/elasticsearch6/util/SwitchUtil.java | 3 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 2 + 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 13cc261ad..04679a039 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -42,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; /** * source data parse to json format @@ -63,6 +64,15 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final List fieldExtraInfos; private final String charsetName; + private static final Pattern TIMESTAMP_PATTERN = Pattern.compile("^\\d+$"); + private static final Pattern TIMESTAMP_FORMAT_PATTERN = Pattern.compile("\\d+-\\d+-\\d+\\s+\\d+:\\d+:\\d+"); + private static final Pattern TIME_FORMAT_PATTERN = Pattern.compile("\\w+\\d+:\\d+:\\d+"); + private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\w+\\d+-\\d+-\\d+"); + + private static final String TIMESTAMP_TYPE = "timestamp"; + private static final String TIME_TYPE = "time"; + private static final String DATE_TYPE = "date"; + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos, String charsetName) { @@ -141,14 +151,14 @@ private Object convert(JsonNode node, TypeInformation info) { return node.asText(); } } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); + return convertToTimestamp(node.asText(), DATE_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { // local zone - return Time.valueOf(node.asText()); + return convertToTimestamp(node.asText(), TIME_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone - return Timestamp.valueOf(node.asText()); - } else if (info instanceof RowTypeInfo) { + return convertToTimestamp(node.asText(), TIMESTAMP_TYPE); + } else if (info instanceof RowTypeInfo) { return convertRow(node, (RowTypeInfo) info); } else if (info instanceof ObjectArrayTypeInfo) { return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); @@ -163,6 +173,38 @@ private Object convert(JsonNode node, TypeInformation info) { } } + /** + * 将 2020-09-07 14:49:10.0 和 1598446699685 两种格式都转化为 Timestamp、Time、Date + */ + private static Object convertToTimestamp(String timestamp, String type) { + if (TIMESTAMP_PATTERN.matcher(timestamp).find()) { + switch (type.toLowerCase()) { + case (TIMESTAMP_TYPE): + return new Timestamp(Long.parseLong(timestamp)); + case (DATE_TYPE): + return new Date(new Timestamp(Long.parseLong(timestamp)).getTime()); + case (TIME_TYPE): + return new Time(new Timestamp(Long.parseLong(timestamp)).getTime()); + default: + throw new RuntimeException(String.format("%s transform to %s error!", timestamp, type)); + } + } + + if (TIMESTAMP_FORMAT_PATTERN.matcher(timestamp).find() && TIMESTAMP_TYPE.equalsIgnoreCase(type)) { + return Timestamp.valueOf(timestamp); + } + + if (TIME_FORMAT_PATTERN.matcher(timestamp).find() && TIME_TYPE.equalsIgnoreCase(type)) { + return Time.valueOf(timestamp); + } + + if (DATE_FORMAT_PATTERN.matcher(timestamp).find() && DATE_TYPE.equalsIgnoreCase(type)) { + return Date.valueOf(timestamp); + } + + throw new IllegalArgumentException("Incorrect time format of timestamp, input: " + timestamp); + } + private Row convertTopRow() { Row row = new Row(fieldNames.length); try { @@ -173,7 +215,7 @@ private Row convertTopRow() { if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { throw new IllegalStateException("Failed to find field with name '" - + fieldNames[i] + "'."); + + fieldNames[i] + "'."); } else { row.setField(i, null); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index b05de0f16..bb48b0678 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -59,6 +59,7 @@ public class DateUtil { private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); + private static final Pattern TIME = Pattern.compile("^\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final int MILLIS_PER_SECOND = 1000; @@ -832,4 +833,16 @@ public static java.sql.Date getDateFromStr(String dateStr) { return null == date ? null : new java.sql.Date(date.getTime()); } + public static java.sql.Time getTimeFromStr(String dateStr) { + if (TIME.matcher(dateStr).matches()) { + dateStr = dateStr.substring(0,dateStr.length()-1); + Instant instant = LocalTime.parse(dateStr).atDate(LocalDate.now()).toInstant(ZoneOffset.UTC); + return new java.sql.Time(instant.toEpochMilli()); + } else if (DATETIME.matcher(dateStr).matches()) { + Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); + return new java.sql.Time(instant.toEpochMilli()); + } + Date date = stringToDate(dateStr); + return null == date ? null : new java.sql.Time(date.getTime()); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index c5584b5d8..844b87033 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -22,7 +22,9 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.util.Objects; /** * Convert val to specified numeric type @@ -234,7 +236,19 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } - + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } public static Timestamp getTimestamp(Object obj) { if (obj == null) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index d72530a56..faf256145 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -19,22 +19,23 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.util.DateUtil; + import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.util.Objects; /** + * Convert val to specified numeric type * Date: 2017/4/21 * Company: www.dtstack.com - * * @author xuchao */ public class MathUtil { - public static Long getLongVal(Object obj) { if (obj == null) { return null; @@ -103,8 +104,6 @@ public static Float getFloatVal(Object obj) { return (Float) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).floatValue(); - } else if (obj instanceof Double) { - return ((Double) obj).floatValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); @@ -126,9 +125,13 @@ public static Double getDoubleVal(Object obj) { if (obj instanceof String) { return Double.valueOf((String) obj); } else if (obj instanceof Float) { + return ((Float) obj).doubleValue(); + } else if (obj instanceof Double) { return (Double) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).doubleValue(); + } else if (obj instanceof Integer) { + return ((Integer) obj).doubleValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); @@ -226,12 +229,7 @@ public static Date getDate(Object obj) { return null; } if (obj instanceof String) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return new Date(format.parse((String) obj).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } + return DateUtil.getDateFromStr((String) obj); } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); } else if (obj instanceof Date) { @@ -240,6 +238,20 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } + public static Timestamp getTimestamp(Object obj) { if (obj == null) { return null; @@ -249,7 +261,7 @@ public static Timestamp getTimestamp(Object obj) { } else if (obj instanceof Date) { return new Timestamp(((Date) obj).getTime()); } else if (obj instanceof String) { - return new Timestamp(getDate(obj).getTime()); + return DateUtil.getTimestampFromStr(obj.toString()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java index e0aaa123e..76165a4a7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -39,7 +39,6 @@ public static Object getTarget(Object obj, String targetType) { case "int": return MathUtil.getIntegerVal(obj); - case "long": case "bigint": case "bigintunsigned": case "intunsigned": @@ -77,6 +76,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: return obj; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 49d163d4d..62390b47d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: } return obj; From 7872c8072d7b5aac393e9b9e9dbffa37a1a34e3f Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 3 Nov 2020 10:44:31 +0800 Subject: [PATCH 337/378] =?UTF-8?q?[fix]=20=E6=94=AF=E6=8C=81=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=88=B3=E8=BD=AC=E5=8C=96=E4=B8=BA=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84Timestamp=E3=80=81Time=E3=80=81Date=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [fix] 支持时间戳转化为对应的Timestamp、Time、Date时间类型 [fix] 支持时间戳转化为对应的Timestamp、Time、Date时间类型 --- .../DtNestRowDeserializationSchema.java | 62 +++++++++++++++++-- .../com/dtstack/flink/sql/util/DateUtil.java | 13 ++++ .../com/dtstack/flink/sql/util/MathUtil.java | 16 ++++- .../side/elasticsearch6/util/MathUtil.java | 38 ++++++++---- .../side/elasticsearch6/util/SwitchUtil.java | 3 +- .../flink/sql/side/rdb/util/SwitchUtil.java | 2 + 6 files changed, 114 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java index 13cc261ad..8b2ca5f49 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java @@ -42,6 +42,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * source data parse to json format @@ -63,6 +65,15 @@ public class DtNestRowDeserializationSchema extends AbstractDeserializationSchem private final List fieldExtraInfos; private final String charsetName; + private static final Pattern TIMESTAMP_PATTERN = Pattern.compile("^\\d+$"); + private static final Pattern TIMESTAMP_FORMAT_PATTERN = Pattern.compile("(\\d+-\\d+-\\d+\\s)(\\d+:\\d+:\\d+)"); + private static final Pattern TIME_FORMAT_PATTERN = Pattern.compile("\\w+\\d+:\\d+:\\d+"); + private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\w+\\d+-\\d+-\\d+"); + + private static final String TIMESTAMP_TYPE = "timestamp"; + private static final String TIME_TYPE = "time"; + private static final String DATE_TYPE = "date"; + public DtNestRowDeserializationSchema(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos, String charsetName) { @@ -141,14 +152,14 @@ private Object convert(JsonNode node, TypeInformation info) { return node.asText(); } } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); + return convertToTimestamp(node.asText(), DATE_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { // local zone - return Time.valueOf(node.asText()); + return convertToTimestamp(node.asText(), TIME_TYPE); } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { // local zone - return Timestamp.valueOf(node.asText()); - } else if (info instanceof RowTypeInfo) { + return convertToTimestamp(node.asText(), TIMESTAMP_TYPE); + } else if (info instanceof RowTypeInfo) { return convertRow(node, (RowTypeInfo) info); } else if (info instanceof ObjectArrayTypeInfo) { return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo()); @@ -163,6 +174,47 @@ private Object convert(JsonNode node, TypeInformation info) { } } + /** + * 将 2020-09-07 14:49:10.0 和 1598446699685 两种格式都转化为 Timestamp、Time、Date + */ + private static Object convertToTimestamp(String timestamp, String type) { + if (TIMESTAMP_PATTERN.matcher(timestamp).find()) { + switch (type.toLowerCase()) { + case (TIMESTAMP_TYPE): + return new Timestamp(Long.parseLong(timestamp)); + case (DATE_TYPE): + return new Date(new Timestamp(Long.parseLong(timestamp)).getTime()); + case (TIME_TYPE): + return new Time(new Timestamp(Long.parseLong(timestamp)).getTime()); + default: + throw new RuntimeException(String.format("%s transform to %s error!", timestamp, type)); + } + } + + Matcher matcher = TIMESTAMP_FORMAT_PATTERN.matcher(timestamp); + if (matcher.find()) { + switch (type.toLowerCase()) { + case TIME_TYPE: + return Time.valueOf(String.valueOf(matcher.group(2)).trim()); + case DATE_TYPE: + return Date.valueOf(String.valueOf(matcher.group(1)).trim()); + case TIMESTAMP_TYPE: + return Timestamp.valueOf(timestamp); + default: + } + } + + if (TIME_FORMAT_PATTERN.matcher(timestamp).find() && TIME_TYPE.equalsIgnoreCase(type)) { + return Time.valueOf(timestamp); + } + + if (DATE_FORMAT_PATTERN.matcher(timestamp).find() && DATE_TYPE.equalsIgnoreCase(type)) { + return Date.valueOf(timestamp); + } + + throw new IllegalArgumentException("Incorrect time format of timestamp, input: " + timestamp); + } + private Row convertTopRow() { Row row = new Row(fieldNames.length); try { @@ -173,7 +225,7 @@ private Row convertTopRow() { if (node == null) { if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { throw new IllegalStateException("Failed to find field with name '" - + fieldNames[i] + "'."); + + fieldNames[i] + "'."); } else { row.setField(i, null); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java index b05de0f16..bb48b0678 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DateUtil.java @@ -59,6 +59,7 @@ public class DateUtil { private static final Pattern DATETIME = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final Pattern DATE = Pattern.compile("^\\d{4}-(?:0[0-9]|1[0-2])-[0-9]{2}$"); + private static final Pattern TIME = Pattern.compile("^\\d{2}:\\d{2}:\\d{2}(\\.\\d{3,9})?Z$"); private static final int MILLIS_PER_SECOND = 1000; @@ -832,4 +833,16 @@ public static java.sql.Date getDateFromStr(String dateStr) { return null == date ? null : new java.sql.Date(date.getTime()); } + public static java.sql.Time getTimeFromStr(String dateStr) { + if (TIME.matcher(dateStr).matches()) { + dateStr = dateStr.substring(0,dateStr.length()-1); + Instant instant = LocalTime.parse(dateStr).atDate(LocalDate.now()).toInstant(ZoneOffset.UTC); + return new java.sql.Time(instant.toEpochMilli()); + } else if (DATETIME.matcher(dateStr).matches()) { + Instant instant = Instant.from(ISO_INSTANT.parse(dateStr)); + return new java.sql.Time(instant.toEpochMilli()); + } + Date date = stringToDate(dateStr); + return null == date ? null : new java.sql.Time(date.getTime()); + } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java index c5584b5d8..844b87033 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/MathUtil.java @@ -22,7 +22,9 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; +import java.util.Objects; /** * Convert val to specified numeric type @@ -234,7 +236,19 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } - + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } public static Timestamp getTimestamp(Object obj) { if (obj == null) { diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java index d72530a56..faf256145 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/MathUtil.java @@ -19,22 +19,23 @@ package com.dtstack.flink.sql.side.elasticsearch6.util; +import com.dtstack.flink.sql.util.DateUtil; + import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.util.Objects; /** + * Convert val to specified numeric type * Date: 2017/4/21 * Company: www.dtstack.com - * * @author xuchao */ public class MathUtil { - public static Long getLongVal(Object obj) { if (obj == null) { return null; @@ -103,8 +104,6 @@ public static Float getFloatVal(Object obj) { return (Float) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).floatValue(); - } else if (obj instanceof Double) { - return ((Double) obj).floatValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Float."); @@ -126,9 +125,13 @@ public static Double getDoubleVal(Object obj) { if (obj instanceof String) { return Double.valueOf((String) obj); } else if (obj instanceof Float) { + return ((Float) obj).doubleValue(); + } else if (obj instanceof Double) { return (Double) obj; } else if (obj instanceof BigDecimal) { return ((BigDecimal) obj).doubleValue(); + } else if (obj instanceof Integer) { + return ((Integer) obj).doubleValue(); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Double."); @@ -226,12 +229,7 @@ public static Date getDate(Object obj) { return null; } if (obj instanceof String) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return new Date(format.parse((String) obj).getTime()); - } catch (ParseException e) { - throw new RuntimeException("String convert to Date fail."); - } + return DateUtil.getDateFromStr((String) obj); } else if (obj instanceof Timestamp) { return new Date(((Timestamp) obj).getTime()); } else if (obj instanceof Date) { @@ -240,6 +238,20 @@ public static Date getDate(Object obj) { throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } + public static Time getTime(Object obj) { + if (Objects.isNull(obj)) { + return null; + } + if (obj instanceof String) { + return DateUtil.getTimeFromStr((String) obj); + } else if (obj instanceof Timestamp) { + return new Time(((Timestamp) obj).getTime()); + } else if (obj instanceof Time) { + return (Time) obj; + } + throw new RuntimeException("not support type of " + obj.getClass() + " convert to Time."); + } + public static Timestamp getTimestamp(Object obj) { if (obj == null) { return null; @@ -249,7 +261,7 @@ public static Timestamp getTimestamp(Object obj) { } else if (obj instanceof Date) { return new Timestamp(((Date) obj).getTime()); } else if (obj instanceof String) { - return new Timestamp(getDate(obj).getTime()); + return DateUtil.getTimestampFromStr(obj.toString()); } throw new RuntimeException("not support type of " + obj.getClass() + " convert to Date."); } diff --git a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java index e0aaa123e..76165a4a7 100644 --- a/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java +++ b/elasticsearch6/elasticsearch6-side/elasticsearch6-side-core/src/main/java/com/dtstack/flink/sql/side/elasticsearch6/util/SwitchUtil.java @@ -39,7 +39,6 @@ public static Object getTarget(Object obj, String targetType) { case "int": return MathUtil.getIntegerVal(obj); - case "long": case "bigint": case "bigintunsigned": case "intunsigned": @@ -77,6 +76,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: return obj; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java index 49d163d4d..62390b47d 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/util/SwitchUtil.java @@ -78,6 +78,8 @@ public static Object getTarget(Object obj, String targetType) { case "timestamp": case "datetime": return MathUtil.getTimestamp(obj); + case "time": + return MathUtil.getTime(obj); default: } return obj; From ab0cbbeeab68bf7e4c44ae5b75fead1d819b8f66 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 3 Nov 2020 17:15:55 +0800 Subject: [PATCH 338/378] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8DHbase=20Times?= =?UTF-8?q?tamp=E7=B1=BB=E5=9E=8B=E6=95=B0=E6=8D=AE=E5=86=99=E5=85=A5?= =?UTF-8?q?=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hbase/rowkeydealer/AbstractRowKeyModeDealer.java | 3 --- .../flink/sql/sink/hbase/HbaseOutputFormat.java | 11 +++++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java index 90ee289bd..6b5d6a24f 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbstractRowKeyModeDealer.java @@ -93,9 +93,6 @@ protected Row fillData(Row input, Object sideInput){ Row row = new Row(outFieldInfoList.size()); for(Map.Entry entry : inFieldIndex.entrySet()){ Object obj = input.getField(entry.getValue()); - if(obj instanceof Timestamp){ - obj = ((Timestamp)obj).getTime(); - } row.setField(entry.getKey(), obj); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index 6c31bd8dd..42a796403 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -189,14 +189,13 @@ private Put getPutByRow(Row record) { Put put = new Put(rowKey.getBytes()); for (int i = 0; i < record.getArity(); ++i) { Object fieldVal = record.getField(i); - byte[] val = null; if (fieldVal != null) { - val = HbaseUtil.toByte(fieldVal); - } - byte[] cf = families[i].getBytes(); - byte[] qualifier = qualifiers[i].getBytes(); + byte[] val = fieldVal.toString().getBytes(); + byte[] cf = families[i].getBytes(); + byte[] qualifier = qualifiers[i].getBytes(); - put.addColumn(cf, qualifier, val); + put.addColumn(cf, qualifier, val); + } } return put; } From ac08db39c56acca87211d4e3d05a77be58b49dbd Mon Sep 17 00:00:00 2001 From: tiezhu Date: Wed, 4 Nov 2020 10:36:40 +0800 Subject: [PATCH 339/378] =?UTF-8?q?[fix]=20=E7=BB=B4=E8=A1=A8JOIN=E5=A4=9A?= =?UTF-8?q?=E4=B8=AA=E5=AD=97=E6=AE=B5=E6=97=A0=E6=95=B0=E6=8D=AE=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java | 2 +- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index 3b6a7f88c..b89eb5615 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -181,7 +181,7 @@ public void asyncInvoke(CRow row, ResultFuture resultFuture) throws Except } private Map parseInputParam(CRow input){ - Map inputParams = Maps.newHashMap(); + Map inputParams = Maps.newLinkedHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.row().getField(conValIndex); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index b3274b80b..e8ef2cd73 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -347,7 +347,7 @@ private void handleQuery(SQLConnection connection, Map inputPara } private Map formatInputParam(Map inputParam) { - Map result = Maps.newHashMap(); + Map result = Maps.newLinkedHashMap(); inputParam.forEach((k, v) -> { result.put(k, convertDataType(v)); }); From f553a3c6d05630ed6ae7b7c052edf33e184753bf Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 5 Nov 2020 16:12:11 +0800 Subject: [PATCH 340/378] =?UTF-8?q?[fix]=20kafka=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=BB=A5'|'=E4=BD=9C=E4=B8=BA=E5=88=86=E5=89=B2=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/format/DeserializationMetricWrapper.java | 3 ++- .../com/dtstack/flink/sql/parser/CreateTableParser.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java index 96ebe09ae..a4addc025 100644 --- a/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java +++ b/core/src/main/java/com/dtstack/flink/sql/format/DeserializationMetricWrapper.java @@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * add metric for source @@ -91,7 +92,7 @@ public void initMetric() { public Row deserialize(byte[] message) throws IOException { try { if (numInRecord.getCount() % dataPrintFrequency == 0) { - LOG.info("receive source data:" + new String(message, "UTF-8")); + LOG.info("receive source data:" + new String(message, StandardCharsets.UTF_8)); } numInRecord.inc(); numInBytes.inc(message.length); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 91b1fd2ac..303295751 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -72,10 +72,10 @@ public void parseSql(String sql, SqlTree sqlTree) { private Map parseProp(String propsStr){ propsStr = propsStr.replaceAll("'\\s*,", "'|"); - String[] strs = propsStr.trim().split("\\|"); + List strings = DtStringUtil.splitIgnoreQuota(propsStr, '|'); Map propMap = Maps.newHashMap(); - for(int i=0; i ss = DtStringUtil.splitIgnoreQuota(strs[i], '='); + for (String str : strings) { + List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); String value = extractValue(ss.get(1).trim()); propMap.put(key, value); From 3a1836c2c37e29e78597440f3f5e110755d8acbe Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 5 Nov 2020 16:44:37 +0800 Subject: [PATCH 341/378] =?UTF-8?q?[fix]=20=E4=BF=AE=E5=A4=8D=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=E9=87=8C','=E8=A2=AB=E8=A7=A3=E6=9E=90=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/parser/CreateTableParser.java | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 303295751..925531240 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -16,8 +16,6 @@ * limitations under the License. */ - - package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.util.DtStringUtil; @@ -41,8 +39,6 @@ public class CreateTableParser implements IParser { private static final Pattern PATTERN = Pattern.compile(PATTERN_STR); - private static final Pattern PROP_PATTERN = Pattern.compile("^'\\s*(.+)\\s*'$"); - public static CreateTableParser newInstance(){ return new CreateTableParser(); } @@ -70,28 +66,19 @@ public void parseSql(String sql, SqlTree sqlTree) { } } - private Map parseProp(String propsStr){ - propsStr = propsStr.replaceAll("'\\s*,", "'|"); - List strings = DtStringUtil.splitIgnoreQuota(propsStr, '|'); + private Map parseProp(String propsStr){ + List strings = DtStringUtil.splitIgnoreQuota(propsStr.trim(), ','); Map propMap = Maps.newHashMap(); for (String str : strings) { List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); - String value = extractValue(ss.get(1).trim()); + String value = ss.get(1).trim().replaceAll("'", "").trim(); propMap.put(key, value); } return propMap; } - private String extractValue(String value) { - Matcher matcher = PROP_PATTERN.matcher(value); - if (matcher.find()) { - return matcher.group(1); - } - throw new RuntimeException("[" + value + "] format is invalid"); - } - public static class SqlParserResult{ private String tableName; From e7610c229a1f36f0e73b406e35e9a6e83cc4091d Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 5 Nov 2020 20:02:10 +0800 Subject: [PATCH 342/378] =?UTF-8?q?[fix]=20add=20jar=20=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E5=8F=8A=E6=B3=A8=E9=87=8A=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/parser/SqlParser.java | 15 +++++++-------- .../com/dtstack/flink/sql/util/DtStringUtil.java | 11 +++-------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index badcf8d9c..f91aa63c4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.parser; @@ -28,7 +28,6 @@ import com.google.common.collect.Lists; import com.google.common.base.Strings; -import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.regex.Matcher; @@ -54,7 +53,7 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){ LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot; } - private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+"); + private static final Pattern ADD_FILE_AND_JAR_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+|(?i).*add\\s+jar\\s+.+"); /** * flink support sql syntax @@ -79,7 +78,7 @@ public static SqlTree parseSql(String sql) throws Exception { .replace("\t", " ").trim(); List sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER); - sqlArr = removeAddFileStmt(sqlArr); + sqlArr = removeAddFileAndJarStmt(sqlArr); SqlTree sqlTree = new SqlTree(); AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser(); for(String childSql : sqlArr){ @@ -162,12 +161,12 @@ public static SqlTree parseSql(String sql) throws Exception { } /** - * remove add file with statment etc. add file /etc/krb5.conf; + * remove add file and jar with statment etc. add file /etc/krb5.conf, add jar xxx.jar; */ - private static List removeAddFileStmt(List stmts) { - List cleanedStmts = new ArrayList<>(); + private static List removeAddFileAndJarStmt(List stmts) { + List cleanedStmts = Lists.newArrayList(); for (String stmt : stmts) { - Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt); + Matcher matcher = ADD_FILE_AND_JAR_PATTERN.matcher(stmt); if(!matcher.matches()) { cleanedStmts.add(stmt); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 3e2ebb82a..f47356c61 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -167,26 +167,20 @@ public static String replaceIgnoreQuota(String str, String oriStr, String replac public static String dealSqlComment(String sql) { boolean inQuotes = false; boolean inSingleQuotes = false; - int bracketLeftNum = 0; StringBuilder b = new StringBuilder(sql.length()); char[] chars = sql.toCharArray(); for (int index = 0; index < chars.length; index ++) { - if (index == chars.length) { - return b.toString(); - } StringBuilder tempSb = new StringBuilder(2); - if (index > 1) { + if (index >= 1) { tempSb.append(chars[index - 1]); tempSb.append(chars[index]); } - if (tempSb.toString().equals("--")) { + if ("--".equals(tempSb.toString())) { if (inQuotes) { b.append(chars[index]); } else if (inSingleQuotes) { b.append(chars[index]); - } else if (bracketLeftNum > 0) { - b.append(chars[index]); } else { b.deleteCharAt(b.length() - 1); while (chars[index] != '\n') { @@ -210,6 +204,7 @@ public static String dealSqlComment(String sql) { return b.toString(); } + public static String col2string(Object column, String type) { String rowData = column.toString(); ColumnType columnType = ColumnType.valueOf(type.toUpperCase()); From b3de9f1fa9c5cb213ca8c9e5f7b585bd33323eb2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 6 Nov 2020 15:09:17 +0800 Subject: [PATCH 343/378] =?UTF-8?q?[opt]=20=E4=BC=98=E5=8C=96impala?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql/sink/impala/ImpalaOutputFormat.java | 89 ++++++++++--------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index a755ee2bf..b3271a36f 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -34,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.rmi.RemoteException; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.DriverManager; @@ -52,7 +51,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -77,7 +75,7 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted - private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; + private static final String[] NEED_QUOTE_TYPE = {"timestamp", "varchar"}; private static final Integer DEFAULT_CONN_TIME_OUT = 60; private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; @@ -162,7 +160,7 @@ public void open(int taskNumber, int numTasks) throws IOException { init(); initMetric(); } catch (Exception e) { - throw new RemoteException("impala output format open error!", e); + throw new RuntimeException("impala output format open error!", e); } } @@ -189,19 +187,23 @@ private void init() throws SQLException { } private void initScheduledTask(Long batchWaitInterval) { - if (batchWaitInterval != 0) { - this.scheduler = new ScheduledThreadPoolExecutor(1, - new DTThreadFactory("impala-upsert-output-format")); - this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { - synchronized (ImpalaOutputFormat.this) { - try { - flush(); - } catch (Exception e) { - LOG.error("Writing records to impala jdbc failed.", e); - throw new RuntimeException("Writing records to impala jdbc failed.", e); + try { + if (batchWaitInterval != 0) { + this.scheduler = new ScheduledThreadPoolExecutor(1, + new DTThreadFactory("impala-upsert-output-format")); + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { + synchronized (ImpalaOutputFormat.this) { + try { + flush(); + } catch (Exception e) { + LOG.error("Writing records to impala jdbc failed.", e); + throw new RuntimeException("Writing records to impala jdbc failed.", e); + } } - } - }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + throw new RuntimeException(e); } } @@ -235,7 +237,7 @@ private void openJdbc() { } } - private synchronized void flush() throws SQLException { + private void flush() throws Exception { if (batchCount > 0) { if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { executeUpdateBatch(); @@ -387,7 +389,7 @@ private String valueConditionAddQuotation(String valueCondition) { } @Override - public synchronized void writeRecord(Tuple2 record) throws IOException { + public void writeRecord(Tuple2 record) throws IOException { try { if (!record.f0) { return; @@ -433,7 +435,7 @@ public synchronized void writeRecord(Tuple2 record) throws IOExcep // Receive data outRecords.inc(); } catch (Exception e) { - throw new RuntimeException("Writing records to impala failed.", e); + throw new IOException("Writing records to impala failed.", e); } } @@ -469,7 +471,7 @@ public void close() throws IOException { updateStatement.close(); } } catch (SQLException e) { - throw new RemoteException("impala connection close failed!"); + throw new RuntimeException("impala connection close failed!", e); } finally { connection = null; statement = null; @@ -489,17 +491,16 @@ public void close() throws IOException { * @param fieldNames field name list * @param partitionFields partition fields * @param rowDataMap row data map - * @throws SQLException throw sql exception */ - private synchronized void executeBatchSql(Statement statement, - String tempSql, - String schema, - String tableName, - String storeType, - Boolean enablePartition, - List fieldNames, - String partitionFields, - Map> rowDataMap) throws SQLException { + private void executeBatchSql(Statement statement, + String tempSql, + String schema, + String tableName, + String storeType, + Boolean enablePartition, + List fieldNames, + String partitionFields, + Map> rowDataMap) { StringBuilder valuesCondition = new StringBuilder(); StringBuilder partitionCondition = new StringBuilder(); String tableFieldsCondition = buildTableFieldsCondition(fieldNames, partitionFields); @@ -510,21 +511,25 @@ private synchronized void executeBatchSql(Statement statement, // kudu ${partitionCondition} is null if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { - rowData = rowDataMap.get(NO_PARTITION); - rowData.forEach(row -> valuesCondition.append(row).append(", ")); - String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) - .replace(PARTITION_CONDITION, partitionCondition.toString()) - .replace(PARTITION_CONSTANT, "") - .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); - String substring = executeSql.substring(0, executeSql.length() - 2); - statement.execute(substring); + try { + rowData = rowDataMap.get(NO_PARTITION); + rowData.forEach(row -> valuesCondition.append(row).append(", ")); + String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) + .replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(PARTITION_CONSTANT, "") + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); + String substring = executeSql.substring(0, executeSql.length() - 2); + statement.execute(substring); + } catch (Exception e) { + throw new RuntimeException("execute impala SQL error!", e); + } return; } // partition sql Set keySet = rowDataMap.keySet(); String finalTempSql = tempSql; - keySet.forEach(key -> { + for (String key : keySet) { try { String executeSql = String.copyValueOf(finalTempSql.toCharArray()); ArrayList valuesConditionList = rowDataMap.get(key); @@ -535,9 +540,9 @@ private synchronized void executeBatchSql(Statement statement, statement.execute(executeSql); partitionCondition.delete(0, partitionCondition.length()); } catch (SQLException sqlException) { - throw new RuntimeException("execute impala partition SQL error! ", sqlException); + throw new RuntimeException("execute impala SQL error! ", sqlException); } - }); + } } /** @@ -583,7 +588,7 @@ private String buildTableFieldsCondition(List fieldNames, String partiti private String buildValuesCondition(List fieldTypes, Row row) { String valuesCondition = fieldTypes.stream().map( f -> { - for(String item : NEED_QUOTE_TYPE) { + for (String item : NEED_QUOTE_TYPE) { if (f.toLowerCase().contains(item)) { return String.format("cast(? as %s)", f.toLowerCase()); } From d733759d56cd04681b3aeb277187a14d346a029b Mon Sep 17 00:00:00 2001 From: tiezhu Date: Tue, 10 Nov 2020 15:58:34 +0800 Subject: [PATCH 344/378] =?UTF-8?q?[fix-32161]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BD=BF=E7=94=A8udf=E8=AF=AD=E6=B3=95=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E7=B1=BB=E5=86=B2=E7=AA=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/dtstack/flink/sql/GetPlan.java | 10 ++++++++++ .../dtstack/flink/sql/exec/ExecuteProcessHelper.java | 8 ++++++-- .../java/com/dtstack/flink/sql/exec/ParamsInfo.java | 12 ++++++++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java index ba6518b3d..409224577 100644 --- a/core/src/main/java/com/dtstack/flink/sql/GetPlan.java +++ b/core/src/main/java/com/dtstack/flink/sql/GetPlan.java @@ -24,6 +24,9 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import java.net.URL; +import java.net.URLClassLoader; + /** * local模式获取sql任务的执行计划 * Date: 2020/2/17 @@ -33,15 +36,22 @@ public class GetPlan { public static String getExecutionPlan(String[] args) { + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader envClassLoader = StreamExecutionEnvironment.class.getClassLoader(); + ClassLoader plannerClassLoader = new URLClassLoader(new URL[0], envClassLoader); try { long start = System.currentTimeMillis(); ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args); + paramsInfo.setGetPlan(true); + Thread.currentThread().setContextClassLoader(plannerClassLoader); StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo); String executionPlan = env.getExecutionPlan(); long end = System.currentTimeMillis(); return ApiResult.createSuccessResultJsonStr(executionPlan, end - start); } catch (Exception e) { return ApiResult.createErrorResultJsonStr(ExceptionUtils.getFullStackTrace(e)); + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5692acf1c..cdfacb733 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -159,7 +159,7 @@ public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInf Map registerTableCache = Maps.newHashMap(); //register udf - ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv); + ExecuteProcessHelper.registerUserDefinedFunction(sqlTree, paramsInfo.getJarUrlList(), tableEnv, paramsInfo.isGetPlan()); //register table schema Set classPathSets = ExecuteProcessHelper.registerTable(sqlTree, env, tableEnv, paramsInfo.getLocalSqlPluginPath(), paramsInfo.getRemoteSqlPluginPath(), paramsInfo.getPluginLoadMode(), sideTableMap, registerTableCache); @@ -245,13 +245,17 @@ private static void sqlTranslation(String localSqlPluginPath, } } - public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv) + public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv, boolean isGetPlan) throws IllegalAccessException, InvocationTargetException { // udf和tableEnv须由同一个类加载器加载 ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); URLClassLoader classLoader = null; List funcList = sqlTree.getFunctionList(); for (CreateFuncParser.SqlParserResult funcInfo : funcList) { + if (isGetPlan) { + classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) currentClassLoader); + } //classloader if (classLoader == null) { classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java index 27cc7702d..9619c75c1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ParamsInfo.java @@ -21,6 +21,7 @@ import java.net.URL; import java.util.List; +import java.util.Objects; import java.util.Properties; /** @@ -39,6 +40,7 @@ public class ParamsInfo { private String pluginLoadMode; private String deployMode; private Properties confProp; + private boolean getPlan = false; public ParamsInfo(String sql, String name, List jarUrlList, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode, String deployMode, Properties confProp) { @@ -52,6 +54,14 @@ public ParamsInfo(String sql, String name, List jarUrlList, String localSql this.confProp = confProp; } + public boolean isGetPlan() { + return getPlan; + } + + public void setGetPlan(boolean getPlan) { + this.getPlan = getPlan; + } + public String getSql() { return sql; } @@ -114,10 +124,8 @@ public static class Builder { private String remoteSqlPluginPath; private String pluginLoadMode; private String deployMode; - private String logLevel; private Properties confProp; - public ParamsInfo.Builder setSql(String sql) { this.sql = sql; return this; From ba0ee350efd4d6356bccae764a91e3657b5b291d Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 12 Nov 2020 20:46:18 +0800 Subject: [PATCH 345/378] [fix-32327][rdb] make rdb connect pool size smaller --- .../com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index ba37c4843..abd7dd50b 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -74,7 +74,7 @@ public class RdbAsyncReqRow extends BaseAsyncReqRow { public final static int DEFAULT_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 20; + public final static int MAX_DB_CONN_POOL_SIZE_LIMIT = 5; public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; From 210dc297329ea1847bfab08ca9f93321285c9ab2 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Fri, 13 Nov 2020 14:08:12 +0800 Subject: [PATCH 346/378] =?UTF-8?q?[fix-31988]\=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dstring=E7=B1=BB=E5=9E=8B=E8=BD=AC=E5=8C=96=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index b3271a36f..35b0c9a50 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -75,7 +75,7 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted - private static final String[] NEED_QUOTE_TYPE = {"timestamp", "varchar"}; + private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; private static final Integer DEFAULT_CONN_TIME_OUT = 60; private static final int RECEIVE_DATA_PRINT_FREQUENCY = 1000; @@ -759,7 +759,5 @@ public ImpalaOutputFormat build() { return format; } - } - } From 47c4a28b1750164f900430a2c6b017662156bc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 17 Nov 2020 14:44:15 +0800 Subject: [PATCH 347/378] =?UTF-8?q?[fix-32324][rdb-all]=E4=BF=AE=E5=A4=8Dr?= =?UTF-8?q?db=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E5=BC=82=E5=B8=B8=EF=BC=8C=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E6=83=85=E5=86=B5=EF=BC=9ADDL=E4=B8=AD=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=B8=BAid,=20name,=20age,=20=E5=8F=AASELECT=20id,=20age?= =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0db947d42..7badb736f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -213,10 +213,11 @@ private void queryAndFillData(Map>> tmpCache, C ResultSet resultSet = statement.executeQuery(sql); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); + String[] fields = sideInfo.getSideTableInfo().getFields(); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); Map sideFieldNamesAndTypes = Maps.newHashMap(); - for (int i = 0; i < sideFieldNames.length; i++) { - sideFieldNamesAndTypes.put(sideFieldNames[i], sideFieldTypes[i]); + for (int i = 0; i < fields.length; i++) { + sideFieldNamesAndTypes.put(fields[i], sideFieldTypes[i]); } while (resultSet.next()) { From d5557748f004967be787df48308eb8502db3d57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Fri, 27 Nov 2020 10:08:45 +0800 Subject: [PATCH 348/378] [fix-32869][rdb]fix rdb task hangs after connect retry. --- .../com/dtstack/flink/sql/util/JDBCUtils.java | 80 +++++++++++++++++++ .../side/rdb/all/AbstractRdbAllReqRow.java | 46 +++++------ 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java index faab22172..42d7b9356 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java @@ -19,9 +19,19 @@ package com.dtstack.flink.sql.util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Objects; public class JDBCUtils { + private static final Logger LOG = LoggerFactory.getLogger(JDBCUtils.class); + private static final Object LOCK = new Object(); public static void forName(String clazz, ClassLoader classLoader) { @@ -44,4 +54,74 @@ public synchronized static void forName(String clazz) { throw new RuntimeException(e); } } + + /** + * 关闭连接资源 + * + * @param rs ResultSet + * @param stmt Statement + * @param conn Connection + * @param commit + */ + public static void closeConnectionResource(ResultSet rs, Statement stmt, Connection conn, boolean commit) { + if (Objects.nonNull(rs)) { + try { + rs.close(); + } catch (SQLException e) { + LOG.warn("Close resultSet error: {}", e.getMessage()); + } + } + + if (Objects.nonNull(stmt)) { + try { + stmt.close(); + } catch (SQLException e) { + LOG.warn("Close statement error:{}", e.getMessage()); + } + } + + if (Objects.nonNull(conn)) { + try { + if (commit) { + commit(conn); + } else { + rollBack(conn); + } + + conn.close(); + } catch (SQLException e) { + LOG.warn("Close connection error:{}", e.getMessage()); + } + } + } + + /** + * 手动提交事物 + * + * @param conn Connection + */ + public static void commit(Connection conn) { + try { + if (!conn.isClosed() && !conn.getAutoCommit()) { + conn.commit(); + } + } catch (SQLException e) { + LOG.warn("commit error:{}", e.getMessage()); + } + } + + /** + * 手动回滚事物 + * + * @param conn Connection + */ + public static void rollBack(Connection conn) { + try { + if (!conn.isClosed() && !conn.getAutoCommit()) { + conn.rollback(); + } + } catch (SQLException e) { + LOG.warn("rollBack error:{}", e.getMessage()); + } + } } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 7badb736f..082784041 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -18,6 +18,7 @@ package com.dtstack.flink.sql.side.rdb.all; +import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.runtime.types.CRow; @@ -167,42 +168,32 @@ protected Object dealTimeAttributeType(Class entry, O boolean isTimeIndicatorTypeInfo = TimeIndicatorTypeInfo.class.isAssignableFrom(entry); if (obj instanceof Timestamp && isTimeIndicatorTypeInfo) { //去除上一层OutputRowtimeProcessFunction 调用时区导致的影响 - obj = ((Timestamp) obj).getTime() + (long)LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); + obj = ((Timestamp) obj).getTime() + (long) LOCAL_TZ.getOffset(((Timestamp) obj).getTime()); } return obj; } private void loadData(Map>> tmpCache) throws SQLException { - RdbSideTableInfo tableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - Connection connection = null; + queryAndFillData(tmpCache, getConnectionWithRetry((RdbSideTableInfo) sideInfo.getSideTableInfo())); + } - try { - for (int i = 0; i < CONN_RETRY_NUM; i++) { + private Connection getConnectionWithRetry(RdbSideTableInfo tableInfo) throws SQLException { + String connInfo = "url:" + tableInfo.getUrl() + "; userName:" + tableInfo.getUserName(); + String errorMsg = null; + for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { + return getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); + } catch (Exception e) { try { - connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); - break; - } catch (Exception e) { - if (i == CONN_RETRY_NUM - 1) { - throw new RuntimeException("", e); - } - try { - String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); - LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); - Thread.sleep(5 * 1000); - } catch (InterruptedException e1) { - LOG.error("", e1); - } + LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); + errorMsg = e.getCause().toString(); + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + LOG.error("", e1); } } - queryAndFillData(tmpCache, connection); - } catch (Exception e) { - LOG.error("", e); - throw new SQLException(e); - } finally { - if (connection != null) { - connection.close(); - } } + throw new SQLException("get conn fail. connInfo: " + connInfo + "\ncause by: " + errorMsg); } private void queryAndFillData(Map>> tmpCache, Connection connection) throws SQLException { @@ -213,8 +204,8 @@ private void queryAndFillData(Map>> tmpCache, C ResultSet resultSet = statement.executeQuery(sql); String[] sideFieldNames = StringUtils.split(sideInfo.getSideSelectFields(), ","); - String[] fields = sideInfo.getSideTableInfo().getFields(); String[] sideFieldTypes = sideInfo.getSideTableInfo().getFieldTypes(); + String[] fields = sideInfo.getSideTableInfo().getFields(); Map sideFieldNamesAndTypes = Maps.newHashMap(); for (int i = 0; i < fields.length; i++) { sideFieldNamesAndTypes.put(fields[i], sideFieldTypes[i]); @@ -236,6 +227,7 @@ private void queryAndFillData(Map>> tmpCache, C tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) .add(oneRow); } + JDBCUtils.closeConnectionResource(resultSet, statement, connection, false); } public int getFetchSize() { From f22586705647a45a820de883bd1f5e5e2f9e3f05 Mon Sep 17 00:00:00 2001 From: chuixue Date: Tue, 1 Dec 2020 15:55:45 +0800 Subject: [PATCH 349/378] [fix-32989][rdb]rdb sink table can not delete --- .../flink/sql/sink/rdb/writer/AbstractUpsertWriter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java index bc4f763a5..9328257a5 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/writer/AbstractUpsertWriter.java @@ -122,7 +122,9 @@ public void addRecord(Tuple2 record) throws SQLException { // we don't need perform a deep copy, because jdbc field are immutable object. Tuple2 tuple2 = objectReuse ? new Tuple2<>(record.f0, Row.copy(record.f1)) : record; // add records to buffer - keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + if (tuple2.f0) { + keyToRows.put(getPrimaryKey(tuple2.f1), tuple2); + } } @Override From 81e6f8e0ed003e646fc4a36bcb8a6ec092a33203 Mon Sep 17 00:00:00 2001 From: xuchao Date: Thu, 3 Dec 2020 11:13:10 +0800 Subject: [PATCH 350/378] =?UTF-8?q?[fix-32973][core]=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8join=20=E5=90=8C=E5=90=8Dfield=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E9=80=BB=E8=BE=91=E4=B8=8D=E4=B8=80=E8=87=B4=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=8F=96=E5=80=BC=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= =?UTF-8?q?=C2=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/BaseAsyncReqRow.java | 2 +- .../flink/sql/side/JoinNodeDealer.java | 17 +++++++------ .../dtstack/flink/sql/util/TableUtils.java | 25 ++++++------------- .../sql/side/rdb/async/RdbAsyncReqRow.java | 2 +- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java index ae8c75f7e..5f85e434e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/BaseAsyncReqRow.java @@ -181,7 +181,7 @@ public void asyncInvoke(CRow row, ResultFuture resultFuture) throws Except } private Map parseInputParam(CRow input){ - Map inputParams = Maps.newHashMap(); + Map inputParams = Maps.newLinkedHashMap(); for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.row().getField(conValIndex); diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java index 6afc12462..a65a53235 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java @@ -285,7 +285,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, SqlBasicCall buildAs = TableUtils.buildAsNodeByJoinInfo(joinInfo, null, null); if(rightIsSide){ - addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentGroupByList, parentWhere, tableRef); + addSideInfoToExeQueue(queueInfo, joinInfo, joinNode, parentSelectList, parentGroupByList, parentWhere, tableRef, fieldRef); } SqlNode newLeftNode = joinNode.getLeft(); @@ -298,7 +298,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode, //替换leftNode 为新的查询 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentGroupByList, parentWhere); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, fieldRef, parentSelectList, parentGroupByList, parentWhere); } return joinInfo; @@ -321,7 +321,8 @@ public void addSideInfoToExeQueue(Queue queueInfo, SqlNodeList parentSelectList, SqlNodeList parentGroupByList, SqlNode parentWhere, - Map tableRef){ + Map tableRef, + Map fieldRef){ //只处理维表 if(!joinInfo.isRightIsSideTable()){ return; @@ -333,7 +334,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, //替换左表为新的表名称 joinNode.setLeft(buildAs); - replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, parentSelectList, parentGroupByList, parentWhere); + replaceSelectAndWhereField(buildAs, leftJoinNode, tableRef, fieldRef, parentSelectList, parentGroupByList, parentWhere); } /** @@ -348,6 +349,7 @@ public void addSideInfoToExeQueue(Queue queueInfo, public void replaceSelectAndWhereField(SqlBasicCall buildAs, SqlNode leftJoinNode, Map tableRef, + Map fieldRef, SqlNodeList parentSelectList, SqlNodeList parentGroupByList, SqlNode parentWhere){ @@ -361,23 +363,22 @@ public void replaceSelectAndWhereField(SqlBasicCall buildAs, } //替换select field 中的对应字段 - HashBiMap fieldReplaceRef = HashBiMap.create(); for(SqlNode sqlNode : parentSelectList.getList()){ for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldRef); } } //TODO 应该根据上面的查询字段的关联关系来替换 //替换where 中的条件相关 for(String tbTmp : fromTableNameSet){ - TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName, fieldReplaceRef); + TableUtils.replaceWhereCondition(parentWhere, tbTmp, newLeftTableName, fieldRef); } if(parentGroupByList != null){ for(SqlNode sqlNode : parentGroupByList.getList()){ for(String tbTmp : fromTableNameSet) { - TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldReplaceRef); + TableUtils.replaceSelectFieldTable(sqlNode, tbTmp, newLeftTableName, fieldRef); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 2aeb06cf8..30ef8b4b3 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -301,7 +301,7 @@ public static void getFromTableInfo(SqlNode fromTable, Set tableNameSet) public static void replaceSelectFieldTable(SqlNode selectNode, String oldTbName, String newTbName, - HashBiMap fieldReplaceRef) { + Map fieldReplaceRef) { if (selectNode.getKind() == AS) { SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; replaceSelectFieldTable(leftNode, oldTbName, newTbName, fieldReplaceRef); @@ -395,22 +395,13 @@ public static void replaceSelectFieldTable(SqlNode selectNode, private static void replaceOneSelectField(SqlIdentifier sqlIdentifier, String newTbName, String oldTbName, - HashBiMap fieldReplaceRef){ + Map fieldReplaceRef){ SqlIdentifier newField = sqlIdentifier.setName(0, newTbName); String fieldName = sqlIdentifier.names.get(1); - String fieldKey = oldTbName + "_" + fieldName; - - if(!fieldReplaceRef.containsKey(fieldKey)){ - if(fieldReplaceRef.inverse().get(fieldName) != null){ - //换一个名字 - String mappingFieldName = ParseUtils.dealDuplicateFieldName(fieldReplaceRef, fieldName); - newField = newField.setName(1, mappingFieldName); - fieldReplaceRef.put(fieldKey, mappingFieldName); - } else { - fieldReplaceRef.put(fieldKey, fieldName); - } - }else { - newField = newField.setName(1, fieldReplaceRef.get(fieldKey)); + String fieldKey = oldTbName + "." + fieldName; + if(fieldReplaceRef.get(fieldKey) != null){ + String newFieldName = fieldReplaceRef.get(fieldKey).split("\\.")[1]; + newField = newField.setName(1, newFieldName); } sqlIdentifier.assignNamesFrom(newField); @@ -511,7 +502,7 @@ public static String getTargetRefField(Map refFieldMap, String c return preFieldName; } - public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName, HashBiMap fieldReplaceRef){ + public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, String newTbName, Map fieldReplaceRef){ if(parentWhere == null){ return; @@ -527,7 +518,7 @@ public static void replaceWhereCondition(SqlNode parentWhere, String oldTbName, } } - private static void replaceConditionNode(SqlNode selectNode, String oldTbName, String newTbName, HashBiMap fieldReplaceRef) { + private static void replaceConditionNode(SqlNode selectNode, String oldTbName, String newTbName, Map fieldReplaceRef) { if(selectNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index abd7dd50b..2bcbd1f28 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -313,7 +313,7 @@ private void handleQuery(SQLConnection connection, Map inputPara } private Map formatInputParam(Map inputParam){ - Map result = Maps.newHashMap(); + Map result = Maps.newLinkedHashMap(); inputParam.forEach((k,v) -> { result.put(k, convertDataType(v)); }); From a4d1b82af371b66f3e38ec36cb1f67986f729e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Tue, 15 Dec 2020 18:58:35 +0800 Subject: [PATCH 351/378] [fix-33268][core] fix hbase rowKey not available due to remove all single quota. rowKey like 'stu'+'_'+sid+'_'+md5(sid). --- .../java/com/dtstack/flink/sql/parser/CreateTableParser.java | 3 ++- .../main/java/com/dtstack/flink/sql/util/DtStringUtil.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index 925531240..1486e8bfa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -20,6 +20,7 @@ import com.dtstack.flink.sql.util.DtStringUtil; import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; @@ -72,7 +73,7 @@ private Map parseProp(String propsStr){ for (String str : strings) { List ss = DtStringUtil.splitIgnoreQuota(str, '='); String key = ss.get(0).trim(); - String value = ss.get(1).trim().replaceAll("'", "").trim(); + String value = DtStringUtil.removeStartAndEndQuota(ss.get(1).trim()); propMap.put(key, value); } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index f47356c61..5af657512 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -408,4 +408,9 @@ public static String getStartQuote() { public static String getEndQuote() { return "\""; } + + public static String removeStartAndEndQuota(String str) { + String removeStart = StringUtils.removeStart(str, "'"); + return StringUtils.removeEnd(removeStart, "'"); + } } From 834e82da46577c75867de73db0f935c178af121f Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 10:13:36 +0800 Subject: [PATCH 352/378] [fix-33847][core][kafka] add array and map types support in the kafka sink. --- .../flink/sql/exec/ExecuteProcessHelper.java | 4 +- .../flink/sql/function/FunctionManager.java | 9 ---- .../com/dtstack/flink/sql/util/ClassUtil.java | 6 +++ .../dtstack/flink/sql/util/DataTypeUtils.java | 45 ++++++++++++------- .../sql/sink/kafka/AbstractKafkaSink.java | 16 ++++++- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index 5692acf1c..b7af4822f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -315,9 +315,7 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); - TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); - tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - + tableEnv.registerTableSink(tableInfo.getName(), tableSink); URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { diff --git a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java index 360cc82ab..5409ac36f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java @@ -122,13 +122,4 @@ public static void registerAggregateUDF(String classPath, String funcName, Table } } - - public static TypeInformation[] transformTypes(Class[] fieldTypes) { - TypeInformation[] types = new TypeInformation[fieldTypes.length]; - for (int i = 0; i < fieldTypes.length; i++) { - types[i] = TypeInformation.of(fieldTypes[i]); - } - - return types; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index 21551364f..aa293616f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -25,6 +25,8 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; /** * Reason: TODO ADD REASON(可选) @@ -41,6 +43,10 @@ public static Class stringConvertClass(String str) { if (lowerStr.startsWith("array")) { return Array.newInstance(Integer.class, 0).getClass(); } + if (lowerStr.startsWith("map")) { + Map m = new HashMap(); + return m.getClass(); + } switch (lowerStr) { case "boolean": diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java index 72a8b4b66..bb15b9c45 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DataTypeUtils.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -30,6 +31,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.commons.lang3.StringUtils.split; + /** * @program: flink.sql * @author: wuren @@ -39,28 +42,13 @@ public class DataTypeUtils { private final static Pattern COMPOSITE_TYPE_PATTERN = Pattern.compile("(.+?)<(.+)>"); private final static String ARRAY = "ARRAY"; + private final static String MAP = "MAP"; private final static String ROW = "ROW"; private final static char FIELD_DELIMITER = ','; private final static char TYPE_DELIMITER = ' '; private DataTypeUtils() {} - /** - * 现在只支持ARRAY类型后续可以加入 MAP等类型 - * @param compositeTypeString - * @return - */ - public static TypeInformation convertToCompositeType(String compositeTypeString) { - Matcher matcher = matchCompositeType(compositeTypeString); - final String errorMsg = "type " + compositeTypeString + "is not support!"; - Preconditions.checkState(matcher.find(), errorMsg); - - String normalizedType = normalizeType(matcher.group(1)); - Preconditions.checkState(ARRAY.equals(normalizedType), errorMsg); - - return convertToArray(compositeTypeString); - } - /** * 目前ARRAY里只支持ROW和其他基本类型 * @param arrayTypeString @@ -86,6 +74,30 @@ public static TypeInformation convertToArray(String arrayTypeString) { return Types.OBJECT_ARRAY(elementType); } + /** + * 目前Map里只支持基本类型 + * @param mapTypeString + * @return + */ + public static TypeInformation convertToMap(String mapTypeString) { + Matcher matcher = matchCompositeType(mapTypeString); + final String errorMsg = mapTypeString + "convert to map type error!"; + Preconditions.checkState(matcher.find(), errorMsg); + + String normalizedType = normalizeType(matcher.group(1)); + Preconditions.checkState(MAP.equals(normalizedType), errorMsg); + + String kvTypeString = matcher.group(2); + String[] kvTypeStringList = StringUtils.split(kvTypeString, ","); + final String mapTypeErrorMsg = "There can only be key and value two types in map declaration."; + Preconditions.checkState(kvTypeStringList.length == 2, mapTypeErrorMsg); + String keyTypeString = normalizeType(kvTypeStringList[0]); + String valueTypeString = normalizeType(kvTypeStringList[1]); + TypeInformation keyType = convertToAtomicType(keyTypeString); + TypeInformation valueType = convertToAtomicType(valueTypeString); + return Types.MAP(keyType, valueType); + } + /** * 目前ROW里只支持基本类型 * @param rowTypeString @@ -104,6 +116,7 @@ public static RowTypeInfo convertToRow(String rowTypeString) { return new RowTypeInfo(info.f0, info.f1); } + private static Tuple2 genFieldInfo(Iterable fieldInfoStrs) { ArrayList types = Lists.newArrayList(); ArrayList fieldNames = Lists.newArrayList(); diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 55d1fa7f8..27699c4f3 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -21,6 +21,7 @@ import com.dtstack.flink.sql.enums.EUpdateMode; import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; +import com.dtstack.flink.sql.util.DataTypeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -38,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.HashMap; import java.util.Optional; import java.util.Properties; import java.util.stream.IntStream; @@ -77,11 +79,21 @@ protected Properties getKafkaProperties(KafkaSinkTableInfo KafkaSinkTableInfo) { } return props; } - + // TODO Source有相同的方法日后可以合并 protected TypeInformation[] getTypeInformations(KafkaSinkTableInfo kafka11SinkTableInfo) { + String[] fieldTypes = kafka11SinkTableInfo.getFieldTypes(); Class[] fieldClasses = kafka11SinkTableInfo.getFieldClasses(); TypeInformation[] types = IntStream.range(0, fieldClasses.length) - .mapToObj(i -> TypeInformation.of(fieldClasses[i])) + .mapToObj( + i -> { + if (fieldClasses[i].isArray()) { + return DataTypeUtils.convertToArray(fieldTypes[i]); + } + if (fieldClasses[i] == new HashMap().getClass()) { + return DataTypeUtils.convertToMap(fieldTypes[i]); + } + return TypeInformation.of(fieldClasses[i]); + }) .toArray(TypeInformation[]::new); return types; } From 8b839324ef2b597f6527ad6f3d595381313b08ae Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 13:47:55 +0800 Subject: [PATCH 353/378] [fix-33847][doc] add kafka sink map type doc. --- docs/plugin/kafkaSink.md | 26 ++++++++++++++++++++++++++ docs/pluginsInfo.md | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/plugin/kafkaSink.md b/docs/plugin/kafkaSink.md index 86b01f3ee..b3a1614bc 100644 --- a/docs/plugin/kafkaSink.md +++ b/docs/plugin/kafkaSink.md @@ -221,3 +221,29 @@ into from MyTable a ``` +## MAP类型示例 +目前Kafka Sink支持Map类型 +```sql +CREATE TABLE ods( + id INT, + name STRING +) WITH ( + ... +); + +CREATE TABLE dwd ( + id INT, + dids MAP> +) WITH ( + type ='kafka', + bootstrapServers ='localhost:9092', + offsetReset ='latest', + groupId='wuren_foo', + topic ='luna_foo', + parallelism ='1' +); + +INSERT INTO dwd + SELECT ods.id, MAP['foo', 1, 'bar', 2] AS dids + FROM ods; +``` diff --git a/docs/pluginsInfo.md b/docs/pluginsInfo.md index 88fda90c9..e9d23cdbd 100644 --- a/docs/pluginsInfo.md +++ b/docs/pluginsInfo.md @@ -1,9 +1,9 @@ ### 1 插件列表 #### 1.1 源表插件 * [kafka 源表插件](plugin/kafkaSource.md) -* [kafka 结果表插件](plugin/kafkaSink.md) #### 1.2 结果表插件 +* [kafka 结果表插件](plugin/kafkaSink.md) * [elasticsearch 结果表插件](plugin/elasticsearchSink.md) * [hbase 结果表插件](plugin/hbaseSink.md) * [mysql 结果表插件](plugin/mysqlSink.md) From cf00b23bf2fb7dfe2e27127cacb9a3f96d70fad3 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 29 Dec 2020 21:41:27 +0800 Subject: [PATCH 354/378] [fix-33847][core] fix other sink except kafka table schema not found problem. --- .../com/dtstack/flink/sql/exec/ExecuteProcessHelper.java | 9 ++++++++- .../com/dtstack/flink/sql/function/FunctionManager.java | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index b7af4822f..df5c6619b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -315,7 +315,14 @@ public static Set registerTable(SqlTree sqlTree, StreamExecutionEnvironment } else if (tableInfo instanceof AbstractTargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((AbstractTargetTableInfo) tableInfo, localSqlPluginPath); - tableEnv.registerTableSink(tableInfo.getName(), tableSink); + // TODO Kafka Sink直接注册，其他的Sink要修复才可以。 + if (tableInfo.getType().startsWith("kafka")) { + tableEnv.registerTableSink(tableInfo.getName(), tableSink); + } else { + TypeInformation[] flinkTypes = FunctionManager.transformTypes(tableInfo.getFieldClasses()); + tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); + } + URL sinkTablePathUrl = PluginUtil.buildSourceAndSinkPathByLoadMode(tableInfo.getType(), AbstractTargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode); pluginClassPathSets.add(sinkTablePathUrl); } else if (tableInfo instanceof AbstractSideTableInfo) { diff --git a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java index 5409ac36f..549276dc2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/function/FunctionManager.java @@ -122,4 +122,12 @@ public static void registerAggregateUDF(String classPath, String funcName, Table } } + public static TypeInformation[] transformTypes(Class[] fieldTypes) { + TypeInformation[] types = new TypeInformation[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + types[i] = TypeInformation.of(fieldTypes[i]); + } + return types; + } + } From d58b2c73a8157eb5b7562ee5e66e497e8b05ae38 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Thu, 31 Dec 2020 10:19:42 +0800 Subject: [PATCH 355/378] =?UTF-8?q?[hotfix-34172][core]=20fix=20flinkPlann?= =?UTF-8?q?er=20=E5=AF=BC=E8=87=B4=E7=9A=84metaspace=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/dtstack/flink/sql/parser/FlinkPlanner.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java index 7c76ec2cd..3f48d5049 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/FlinkPlanner.java @@ -36,12 +36,8 @@ private FlinkPlanner() { } public static FlinkPlannerImpl createFlinkPlanner(FrameworkConfig frameworkConfig, RelOptPlanner relOptPlanner, FlinkTypeFactory typeFactory) { - if (flinkPlanner == null) { - synchronized (FlinkPlanner.class) { - if (flinkPlanner == null) { - flinkPlanner = new FlinkPlannerImpl(frameworkConfig, relOptPlanner, typeFactory); - } - } + synchronized (FlinkPlanner.class) { + flinkPlanner = new FlinkPlannerImpl(frameworkConfig, relOptPlanner, typeFactory); } return flinkPlanner; } From 7de4d0432cba027dc28058576dd7a186b11334a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 31 Dec 2020 11:14:36 +0800 Subject: [PATCH 356/378] =?UTF-8?q?[fix-34156][rdb]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E4=B8=BB=E9=94=AE=E5=AD=98?= =?UTF-8?q?=E5=9C=A8null=E5=80=BC=E6=97=B6=E5=87=BA=E7=8E=B0=E7=A9=BA?= =?UTF-8?q?=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 0530e57e2..1f8cc2a7f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -225,7 +225,7 @@ private void queryAndFillData(Map>> tmpCache, C String cacheKey = sideInfo.getEqualFieldList().stream() .map(oneRow::get) - .map(Object::toString) + .map(String::valueOf) .collect(Collectors.joining("_")); tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) From 50dc15b64492bf9a70e5acb4f029a7fb25c6f007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 31 Dec 2020 11:14:36 +0800 Subject: [PATCH 357/378] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=85=A8=E9=87=8F?= =?UTF-8?q?=E7=BB=B4=E8=A1=A8=E4=B8=BB=E9=94=AE=E5=AD=98=E5=9C=A8null?= =?UTF-8?q?=E5=80=BC=E6=97=B6=E5=87=BA=E7=8E=B0=E7=A9=BA=E6=8C=87=E9=92=88?= =?UTF-8?q?=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java index 082784041..853060755 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/AbstractRdbAllReqRow.java @@ -221,7 +221,7 @@ private void queryAndFillData(Map>> tmpCache, C String cacheKey = sideInfo.getEqualFieldList().stream() .map(oneRow::get) - .map(Object::toString) + .map(String::valueOf) .collect(Collectors.joining("_")); tmpCache.computeIfAbsent(cacheKey, key -> Lists.newArrayList()) From 1b7936107963d1ebb65a6c1913f0c262c90505b4 Mon Sep 17 00:00:00 2001 From: HiLany Date: Mon, 4 Jan 2021 16:05:02 +0800 Subject: [PATCH 358/378] [fix-33981][elasticsearch6][sink]quota RequestLogger.class source code, adjust log level to debug when httpResponse.getHeaders has warning. --- .../elasticsearch/client/RequestLogger.java | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java diff --git a/elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java b/elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java new file mode 100644 index 000000000..39d79e03a --- /dev/null +++ b/elasticsearch6/elasticsearch6-sink/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -0,0 +1,180 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.RequestLine; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.BufferedHttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.util.EntityUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Quota this source code in 2021-01-04. + * Quota Reason: http://redmine.prod.dtstack.cn/issues/33981 + * Modify Content: adjust log level to debug when httpResponse.getHeaders has warning. + * + * Helper class that exposes static methods to unify the way requests are logged. + * Includes trace logging to log complete requests and responses in curl format. + * Useful for debugging, manually sending logged requests via curl and checking their responses. + * Trace logging is a feature that all the language clients provide. + */ +final class RequestLogger { + + private static final Log tracer = LogFactory.getLog("tracer"); + + private RequestLogger() { + } + + /** + * Logs a request that yielded a response + */ + static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) { + if (logger.isDebugEnabled()) { + logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + + "] returned [" + httpResponse.getStatusLine() + "]"); + } + // adjust log level to debug when httpResponse.getHeaders has warning. + if (logger.isDebugEnabled()) { + Header[] warnings = httpResponse.getHeaders("Warning"); + if (warnings != null && warnings.length > 0) { + logger.debug(buildWarningMessage(request, host, warnings)); + } + } + if (tracer.isTraceEnabled()) { + String requestLine; + try { + requestLine = buildTraceRequest(request, host); + } catch(IOException e) { + requestLine = ""; + tracer.trace("error while reading request for trace purposes", e); + } + String responseLine; + try { + responseLine = buildTraceResponse(httpResponse); + } catch(IOException e) { + responseLine = ""; + tracer.trace("error while reading response for trace purposes", e); + } + tracer.trace(requestLine + '\n' + responseLine); + } + } + + /** + * Logs a request that failed + */ + static void logFailedRequest(Log logger, HttpUriRequest request, Node node, Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("request [" + request.getMethod() + " " + node.getHost() + getUri(request.getRequestLine()) + "] failed", e); + } + if (tracer.isTraceEnabled()) { + String traceRequest; + try { + traceRequest = buildTraceRequest(request, node.getHost()); + } catch (IOException e1) { + tracer.trace("error while reading request for trace purposes", e); + traceRequest = ""; + } + tracer.trace(traceRequest); + } + } + + static String buildWarningMessage(HttpUriRequest request, HttpHost host, Header[] warnings) { + StringBuilder message = new StringBuilder("request [").append(request.getMethod()).append(" ").append(host) + .append(getUri(request.getRequestLine())).append("] returned ").append(warnings.length).append(" warnings: "); + for (int i = 0; i < warnings.length; i++) { + if (i > 0) { + message.append(","); + } + message.append("[").append(warnings[i].getValue()).append("]"); + } + return message.toString(); + } + + /** + * Creates curl output for given request + */ + static String buildTraceRequest(HttpUriRequest request, HttpHost host) throws IOException { + String requestLine = "curl -iX " + request.getMethod() + " '" + host + getUri(request.getRequestLine()) + "'"; + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request; + if (enclosingRequest.getEntity() != null) { + requestLine += " -d '"; + HttpEntity entity = enclosingRequest.getEntity(); + if (entity.isRepeatable() == false) { + entity = new BufferedHttpEntity(enclosingRequest.getEntity()); + enclosingRequest.setEntity(entity); + } + requestLine += EntityUtils.toString(entity, StandardCharsets.UTF_8) + "'"; + } + } + return requestLine; + } + + /** + * Creates curl output for given response + */ + static String buildTraceResponse(HttpResponse httpResponse) throws IOException { + StringBuilder responseLine = new StringBuilder(); + responseLine.append("# ").append(httpResponse.getStatusLine()); + for (Header header : httpResponse.getAllHeaders()) { + responseLine.append("\n# ").append(header.getName()).append(": ").append(header.getValue()); + } + responseLine.append("\n#"); + HttpEntity entity = httpResponse.getEntity(); + if (entity != null) { + if (entity.isRepeatable() == false) { + entity = new BufferedHttpEntity(entity); + } + httpResponse.setEntity(entity); + ContentType contentType = ContentType.get(entity); + Charset charset = StandardCharsets.UTF_8; + if (contentType != null && contentType.getCharset() != null) { + charset = contentType.getCharset(); + } + try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) { + String line; + while( (line = reader.readLine()) != null) { + responseLine.append("\n# ").append(line); + } + } + } + return responseLine.toString(); + } + + private static String getUri(RequestLine requestLine) { + if (requestLine.getUri().charAt(0) != '/') { + return "/" + requestLine.getUri(); + } + return requestLine.getUri(); + } +} From ec96b760638b8b248ef9eed0e1981ef57a485ecd Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 18:41:04 +0800 Subject: [PATCH 359/378] [fix-34344][kafka] remove partition key blank char. --- .../flink/sql/sink/kafka/AbstractKafkaSink.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 27699c4f3..89be2840b 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -39,6 +39,7 @@ import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.Arrays; import java.util.HashMap; import java.util.Optional; import java.util.Properties; @@ -128,8 +129,13 @@ public CRowTypeInfo getRowTypeInfo() { } protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { - if (StringUtils.isNotBlank(kafkaSinkTableInfo.getPartitionKeys())) { - return StringUtils.split(kafkaSinkTableInfo.getPartitionKeys(), ','); + String keysStr = kafkaSinkTableInfo.getPartitionKeys(); + if (StringUtils.isNotBlank(keysStr)) { + String[] keys = keysStr.split(","); + String[] cleanedKeys = Arrays.stream(keys) + .map(x -> x.trim()) + .toArray(String[]::new); + return cleanedKeys; } return null; } From 410c5e0e7c39e25930f1db2858250dbfb912326a Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 19:31:54 +0800 Subject: [PATCH 360/378] [fix-34344][kafka] use apache common split --- .../com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java index 89be2840b..0a27cb074 100644 --- a/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java +++ b/kafka-base/kafka-base-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/AbstractKafkaSink.java @@ -131,7 +131,7 @@ public CRowTypeInfo getRowTypeInfo() { protected String[] getPartitionKeys(KafkaSinkTableInfo kafkaSinkTableInfo) { String keysStr = kafkaSinkTableInfo.getPartitionKeys(); if (StringUtils.isNotBlank(keysStr)) { - String[] keys = keysStr.split(","); + String[] keys = StringUtils.split(keysStr, ","); String[] cleanedKeys = Arrays.stream(keys) .map(x -> x.trim()) .toArray(String[]::new); From 69e9b1fb5ee4a8039002341394b45f9fd3bc1c08 Mon Sep 17 00:00:00 2001 From: wuren Date: Tue, 5 Jan 2021 21:01:04 +0800 Subject: [PATCH 361/378] [fix-34193][root][pom] delete plugins dir in maven clean phase --- pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pom.xml b/pom.xml index d10954d3e..f2e85ea33 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,25 @@ + + maven-antrun-plugin + 1.8 + + + clean-plugins + + clean + + run + + + + + + + + + From 708d9235e40e978da7234a058f705e2cd22e697a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Wed, 6 Jan 2021 16:08:05 +0800 Subject: [PATCH 362/378] =?UTF-8?q?[opt-34257][kudu]=20=E4=BC=98=E5=8C=96k?= =?UTF-8?q?udu=E6=8F=92=E4=BB=B6=EF=BC=8C=E6=94=B9=E7=94=A8KuduClient=20AP?= =?UTF-8?q?I=E4=BB=A3=E6=9B=BFAsyncKuduClient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/sink/kudu/KuduOutputFormat.java | 272 +++++++++--------- .../dtstack/flink/sql/sink/kudu/KuduSink.java | 21 +- 2 files changed, 140 insertions(+), 153 deletions(-) diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java index ab6ac13a8..e8b508f1f 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java @@ -25,10 +25,9 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.kudu.client.AsyncKuduClient; -import org.apache.kudu.client.AsyncKuduSession; import org.apache.kudu.client.KuduClient; import org.apache.kudu.client.KuduException; +import org.apache.kudu.client.KuduSession; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.Operation; import org.apache.kudu.client.PartialRow; @@ -40,40 +39,29 @@ import java.security.PrivilegedAction; import java.sql.Timestamp; import java.util.Date; +import java.util.Objects; /** - * @author gituser - * @modify xiuzhu + * @author gituser + * @modify xiuzhu */ public class KuduOutputFormat extends AbstractDtRichOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(KuduOutputFormat.class); - - public enum WriteMode { - // insert - INSERT, - // update - UPDATE, - // update or insert - UPSERT - } - + protected String[] fieldNames; + TypeInformation[] fieldTypes; + boolean enableKrb; private String kuduMasters; - private String tableName; - private WriteMode writeMode; - - protected String[] fieldNames; - - TypeInformation[] fieldTypes; - - private AsyncKuduClient client; + private KuduClient client; private KuduTable table; + private volatile KuduSession session; + private Integer workerCount; private Integer defaultOperationTimeoutMs; @@ -86,11 +74,14 @@ public enum WriteMode { private String principal; private String keytab; private String krb5conf; - boolean enableKrb; private KuduOutputFormat() { } + public static KuduOutputFormatBuilder buildKuduOutputFormat() { + return new KuduOutputFormatBuilder(); + } + @Override public void configure(Configuration parameters) { @@ -103,16 +94,16 @@ public void open(int taskNumber, int numTasks) throws IOException { } private void establishConnection() throws IOException { - AsyncKuduClient.AsyncKuduClientBuilder asyncKuduClientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasters); + KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(kuduMasters); if (null != workerCount) { - asyncKuduClientBuilder.workerCount(workerCount); + kuduClientBuilder.workerCount(workerCount); } if (null != defaultSocketReadTimeoutMs) { - asyncKuduClientBuilder.workerCount(defaultSocketReadTimeoutMs); + kuduClientBuilder.defaultSocketReadTimeoutMs(defaultSocketReadTimeoutMs); } if (null != defaultOperationTimeoutMs) { - asyncKuduClientBuilder.workerCount(defaultOperationTimeoutMs); + kuduClientBuilder.defaultOperationTimeoutMs(defaultOperationTimeoutMs); } if (enableKrb) { @@ -122,20 +113,21 @@ private void establishConnection() throws IOException { krb5conf ); client = ugi.doAs( - new PrivilegedAction() { - @Override - public AsyncKuduClient run() { - return asyncKuduClientBuilder.build(); - } - }); + (PrivilegedAction) kuduClientBuilder::build); } else { - client = asyncKuduClientBuilder.build(); + client = kuduClientBuilder.build(); + } + + if (client.tableExists(tableName)) { + table = client.openTable(tableName); } - LOG.info("connect kudu is successed!"); - KuduClient syncClient = client.syncClient(); - if (syncClient.tableExists(tableName)) { - table = syncClient.openTable(tableName); + if (Objects.isNull(table)) { + throw new IllegalArgumentException( + String.format("Table [%s] Open Failed , please check table exists", tableName)); } + LOG.info("connect kudu is succeed!"); + + session = client.newSession(); } @Override @@ -147,26 +139,22 @@ public void writeRecord(Tuple2 record) throws IOException { } Row row = tupleTrans.getField(1); if (row.getArity() != fieldNames.length) { - if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { + if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { LOG.error("record insert failed ..{}", row.toString()); LOG.error("cause by row.getArity() != fieldNames.length"); } outDirtyRecords.inc(); return; } - Operation operation = toOperation(writeMode, row); - AsyncKuduSession session = client.newSession(); try { if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0) { LOG.info("Receive data : {}", row); } - - session.apply(operation); - session.close(); + session.apply(toOperation(writeMode, row)); outRecords.inc(); } catch (KuduException e) { - if(outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0){ + if (outDirtyRecords.getCount() % DIRTY_PRINT_FREQUENCY == 0) { LOG.error("record insert failed, total dirty record:{} current row:{}", outDirtyRecords.getCount(), row.toString()); LOG.error("", e); } @@ -176,107 +164,24 @@ public void writeRecord(Tuple2 record) throws IOException { @Override public void close() { - if (null != client) { + if (Objects.nonNull(session) && !session.isClosed()) { try { - client.close(); + session.close(); } catch (Exception e) { - throw new IllegalArgumentException("[closeKudu]:" + e.getMessage()); + throw new IllegalArgumentException("[closeKuduSession]: " + e.getMessage()); } } - } - - public static KuduOutputFormatBuilder buildKuduOutputFormat() { - return new KuduOutputFormatBuilder(); - } - - public static class KuduOutputFormatBuilder { - private final KuduOutputFormat kuduOutputFormat; - - protected KuduOutputFormatBuilder() { - this.kuduOutputFormat = new KuduOutputFormat(); - } - - public KuduOutputFormatBuilder setKuduMasters(String kuduMasters) { - kuduOutputFormat.kuduMasters = kuduMasters; - return this; - } - - public KuduOutputFormatBuilder setTableName(String tableName) { - kuduOutputFormat.tableName = tableName; - return this; - } - - - public KuduOutputFormatBuilder setFieldNames(String[] fieldNames) { - kuduOutputFormat.fieldNames = fieldNames; - return this; - } - - public KuduOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes) { - kuduOutputFormat.fieldTypes = fieldTypes; - return this; - } - - public KuduOutputFormatBuilder setWriteMode(WriteMode writeMode) { - if (null == writeMode) { - kuduOutputFormat.writeMode = WriteMode.UPSERT; - } - kuduOutputFormat.writeMode = writeMode; - return this; - } - - public KuduOutputFormatBuilder setWorkerCount(Integer workerCount) { - kuduOutputFormat.workerCount = workerCount; - return this; - } - - public KuduOutputFormatBuilder setDefaultOperationTimeoutMs(Integer defaultOperationTimeoutMs) { - kuduOutputFormat.defaultOperationTimeoutMs = defaultOperationTimeoutMs; - return this; - } - - public KuduOutputFormatBuilder setDefaultSocketReadTimeoutMs(Integer defaultSocketReadTimeoutMs) { - kuduOutputFormat.defaultSocketReadTimeoutMs = defaultSocketReadTimeoutMs; - return this; - } - - public KuduOutputFormatBuilder setPrincipal(String principal) { - kuduOutputFormat.principal = principal; - return this; - } - - public KuduOutputFormatBuilder setKeytab(String keytab) { - kuduOutputFormat.keytab = keytab; - return this; - } - - public KuduOutputFormatBuilder setKrb5conf(String krb5conf) { - kuduOutputFormat.krb5conf = krb5conf; - return this; - } - - public KuduOutputFormatBuilder setEnableKrb(boolean enableKrb) { - kuduOutputFormat.enableKrb = enableKrb; - return this; - } - public KuduOutputFormat finish() { - if (kuduOutputFormat.kuduMasters == null) { - throw new IllegalArgumentException("No kuduMasters supplied."); - } - - if (kuduOutputFormat.tableName == null) { - throw new IllegalArgumentException("No tablename supplied."); + if (null != client) { + try { + client.shutdown(); + } catch (Exception e) { + throw new IllegalArgumentException("[closeKuduClient]:" + e.getMessage()); } - - return kuduOutputFormat; } } private Operation toOperation(WriteMode writeMode, Row row) { - if (null == table) { - throw new IllegalArgumentException("Table Open Failed , please check table exists"); - } Operation operation = toOperation(writeMode); PartialRow partialRow = operation.getRow(); @@ -355,11 +260,102 @@ private Operation toOperation(WriteMode writeMode) { return table.newInsert(); case UPDATE: return table.newUpdate(); - case UPSERT: - return table.newUpsert(); default: return table.newUpsert(); } } + public enum WriteMode { + // insert + INSERT, + // update + UPDATE, + // update or insert + UPSERT + } + + public static class KuduOutputFormatBuilder { + private final KuduOutputFormat kuduOutputFormat; + + protected KuduOutputFormatBuilder() { + this.kuduOutputFormat = new KuduOutputFormat(); + } + + public KuduOutputFormatBuilder setKuduMasters(String kuduMasters) { + kuduOutputFormat.kuduMasters = kuduMasters; + return this; + } + + public KuduOutputFormatBuilder setTableName(String tableName) { + kuduOutputFormat.tableName = tableName; + return this; + } + + + public KuduOutputFormatBuilder setFieldNames(String[] fieldNames) { + kuduOutputFormat.fieldNames = fieldNames; + return this; + } + + public KuduOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes) { + kuduOutputFormat.fieldTypes = fieldTypes; + return this; + } + + public KuduOutputFormatBuilder setWriteMode(WriteMode writeMode) { + if (null == writeMode) { + kuduOutputFormat.writeMode = WriteMode.UPSERT; + } + kuduOutputFormat.writeMode = writeMode; + return this; + } + + public KuduOutputFormatBuilder setWorkerCount(Integer workerCount) { + kuduOutputFormat.workerCount = workerCount; + return this; + } + + public KuduOutputFormatBuilder setDefaultOperationTimeoutMs(Integer defaultOperationTimeoutMs) { + kuduOutputFormat.defaultOperationTimeoutMs = defaultOperationTimeoutMs; + return this; + } + + public KuduOutputFormatBuilder setDefaultSocketReadTimeoutMs(Integer defaultSocketReadTimeoutMs) { + kuduOutputFormat.defaultSocketReadTimeoutMs = defaultSocketReadTimeoutMs; + return this; + } + + public KuduOutputFormatBuilder setPrincipal(String principal) { + kuduOutputFormat.principal = principal; + return this; + } + + public KuduOutputFormatBuilder setKeytab(String keytab) { + kuduOutputFormat.keytab = keytab; + return this; + } + + public KuduOutputFormatBuilder setKrb5conf(String krb5conf) { + kuduOutputFormat.krb5conf = krb5conf; + return this; + } + + public KuduOutputFormatBuilder setEnableKrb(boolean enableKrb) { + kuduOutputFormat.enableKrb = enableKrb; + return this; + } + + public KuduOutputFormat finish() { + if (kuduOutputFormat.kuduMasters == null) { + throw new IllegalArgumentException("No kuduMasters supplied."); + } + + if (kuduOutputFormat.tableName == null) { + throw new IllegalArgumentException("No tablename supplied."); + } + + return kuduOutputFormat; + } + } + } \ No newline at end of file diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java index f8ac125be..cd87e3fbf 100644 --- a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java +++ b/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java @@ -20,28 +20,19 @@ public class KuduSink implements RetractStreamTableSink, Serializable, IStreamSinkGener { + protected String[] fieldNames; + TypeInformation[] fieldTypes; + boolean enableKrb; private String kuduMasters; - private String tableName; - private KuduOutputFormat.WriteMode writeMode; - - protected String[] fieldNames; - - TypeInformation[] fieldTypes; - private Integer workerCount; - private Integer defaultOperationTimeoutMs; - private Integer defaultSocketReadTimeoutMs; - private int parallelism = -1; - private String principal; private String keytab; private String krb5conf; - boolean enableKrb; @Override public KuduSink genStreamSink(AbstractTargetTableInfo targetTableInfo) { @@ -67,7 +58,7 @@ public void emitDataStream(DataStream> dataStream) { public DataStreamSink> consumeDataStream(DataStream> dataStream) { KuduOutputFormat.KuduOutputFormatBuilder builder = KuduOutputFormat.buildKuduOutputFormat(); - builder.setKuduMasters(this.kuduMasters) + KuduOutputFormat kuduOutputFormat = builder.setKuduMasters(this.kuduMasters) .setTableName(this.tableName) .setWriteMode(writeMode) .setWorkerCount(this.workerCount) @@ -78,8 +69,8 @@ public DataStreamSink> consumeDataStream(DataStream Date: Wed, 6 Jan 2021 18:51:14 +0800 Subject: [PATCH 363/378] [hotfix-33981][elasticsearch6][sink]remove logback dependence and exclude jar from 'flink-connector-elasticsearch6_2.11' --- elasticsearch6/elasticsearch6-sink/pom.xml | 24 ++++++---------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml index cb5fb753c..c1e755b2d 100644 --- a/elasticsearch6/elasticsearch6-sink/pom.xml +++ b/elasticsearch6/elasticsearch6-sink/pom.xml @@ -30,28 +30,16 @@ ${elasticsearch.version} - - ch.qos.logback - logback-core - 1.1.7 - - - - ch.qos.logback - logback-classic - 1.1.7 - - - - org.apache.logging.log4j - log4j-to-slf4j - 2.7 - - org.apache.flink flink-connector-elasticsearch6_2.11 ${flink.version} + + + log4j-to-slf4j + org.apache.logging.log4j + + From 081b0c2a1fcb8e817217312e60649667c59663bc Mon Sep 17 00:00:00 2001 From: wuren Date: Wed, 6 Jan 2021 19:45:55 +0800 Subject: [PATCH 364/378] [fix-34400][root][pom] optimize pom for cleaning output dir --- pom.xml | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index f2e85ea33..1b797690d 100644 --- a/pom.xml +++ b/pom.xml @@ -138,23 +138,19 @@ - maven-antrun-plugin - 1.8 - - - clean-plugins - - clean - - run - - - - - - - - + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + ${basedir}/plugins + + + ${basedir}/sqlplugins + + + From 4388684ea2c16bf4effd15c214a5ded2a6cc6de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=93=81=E6=9F=B1?= Date: Thu, 7 Jan 2021 17:59:01 +0800 Subject: [PATCH 365/378] =?UTF-8?q?[fix-34473][core]=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=85=A8=E5=B1=80=E5=B9=B6=E8=A1=8C=E5=BA=A6?= =?UTF-8?q?=E5=90=8E=EF=BC=8C=E5=85=A8=E9=87=8F=E7=BB=B4=E8=A1=A8=E5=8D=95?= =?UTF-8?q?=E7=8B=AC=E8=AE=BE=E7=BD=AE=E5=B9=B6=E8=A1=8C=E5=BA=A6=E4=B8=8D?= =?UTF-8?q?=E7=94=9F=E6=95=88=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/side/operator/SideWithAllCacheOperator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 6b6f9fe1b..f825180e4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -60,6 +60,6 @@ private static BaseAllReqRow loadFlatMap(String sideType, String sqlRootDir, Row public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) throws Exception { BaseAllReqRow allReqRow = loadFlatMap(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - return inputStream.flatMap(allReqRow); + return inputStream.flatMap(allReqRow).setParallelism(sideTableInfo.getParallelism()); } } From b720842135014e93e871326ccfece24afd2d386b Mon Sep 17 00:00:00 2001 From: mowen Date: Fri, 8 Jan 2021 11:27:07 +0800 Subject: [PATCH 366/378] [hotfix-34455][core]verify whether join's columns exists in table. --- .../dtstack/flink/sql/side/SideSqlExec.java | 83 +++++++++++++++++-- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index bb8faf8ee..6dad92061 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -60,12 +60,7 @@ import org.slf4j.LoggerFactory; import java.sql.Timestamp; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; +import java.util.*; import static org.apache.calcite.sql.SqlKind.*; @@ -351,7 +346,7 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, FieldReplaceInfo fieldReplaceInfo = parseAsQuery((SqlBasicCall) pollSqlNode, tableCache); if(fieldReplaceInfo == null){ - return; + return; } //as 的源表 @@ -361,6 +356,77 @@ protected void dealAsSourceTable(StreamTableEnvironment tableEnv, } + /** + * check whether all table fields exist in join condition. + * @param conditionNode + * @param joinScope + */ + public void checkConditionFieldsInTable(SqlNode conditionNode, JoinScope joinScope, AbstractSideTableInfo sideTableInfo) { + List sqlNodeList = Lists.newArrayList(); + ParseUtils.parseAnd(conditionNode, sqlNodeList); + for (SqlNode sqlNode : sqlNodeList) { + if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { + throw new RuntimeException("It is not comparison operator."); + } + + SqlNode leftNode = ((SqlBasicCall) sqlNode).getOperands()[0]; + SqlNode rightNode = ((SqlBasicCall) sqlNode).getOperands()[1]; + + if (leftNode.getKind() == SqlKind.IDENTIFIER) { + checkFieldInTable((SqlIdentifier) leftNode, joinScope, conditionNode, sideTableInfo); + } + + if (rightNode.getKind() == SqlKind.IDENTIFIER) { + checkFieldInTable((SqlIdentifier) rightNode, joinScope, conditionNode, sideTableInfo); + } + + } + } + + /** + * check whether table exists and whether field is in table. + * @param sqlNode + * @param joinScope + * @param conditionNode + */ + private void checkFieldInTable(SqlIdentifier sqlNode, JoinScope joinScope, SqlNode conditionNode, AbstractSideTableInfo sideTableInfo) { + String tableName = sqlNode.getComponent(0).getSimple(); + String fieldName = sqlNode.getComponent(1).getSimple(); + JoinScope.ScopeChild scopeChild = joinScope.getScope(tableName); + String tableErrorMsg = "Table [%s] is not exist. Error condition is [%s]. If you find [%s] is exist. Please check AS statement."; + Preconditions.checkState( + scopeChild != null, + tableErrorMsg, + tableName, + conditionNode.toString(), + tableName + ); + + String[] fieldNames = scopeChild.getRowTypeInfo().getFieldNames(); + ArrayList allFieldNames = new ArrayList( + Arrays.asList(fieldNames) + ); + // HBase、Redis这种NoSQL Primary Key不在字段列表中，所以要加进去。 + if (sideTableInfo != null) { + List pks = sideTableInfo.getPrimaryKeys(); + if (pks != null) { + pks.stream() + .filter(pk -> !allFieldNames.contains(pk)) + .forEach(pk -> allFieldNames.add(pk)); + } + } + + boolean hasField = allFieldNames.contains(fieldName); + String fieldErrorMsg = "Table [%s] has not [%s] field. Error join condition is [%s]. If you find it is exist. Please check AS statement."; + Preconditions.checkState( + hasField, + fieldErrorMsg, + tableName, + fieldName, + conditionNode.toString() + ); + } + private void joinFun(Object pollObj, Map localTableCache, Map sideTableMap, @@ -395,6 +461,9 @@ private void joinFun(Object pollObj, HashBasedTable mappingTable = ((JoinInfo) pollObj).getTableFieldRef(); + // verify whether join's columns exists in table. + checkConditionFieldsInTable(joinInfo.getCondition(), joinScope, sideTableInfo); + //获取两个表的所有字段 List sideJoinFieldInfo = ParserJoinField.getRowTypeInfo(joinInfo.getSelectNode(), joinScope, true); //通过join的查询字段信息过滤出需要的字段信息 From 8009e35325b6ecb42704e98a50f1fe200e4007a8 Mon Sep 17 00:00:00 2001 From: mowen Date: Mon, 18 Jan 2021 15:24:19 +0800 Subject: [PATCH 367/378] [hotfix-34801][core]fix proplem when replace alias and fieldName --- .../com/dtstack/flink/sql/util/TableUtils.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java index 30ef8b4b3..9d11014bf 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/TableUtils.java @@ -528,11 +528,25 @@ private static void replaceConditionNode(SqlNode selectNode, String oldTbName, S String tableName = sqlIdentifier.names.asList().get(0); String tableField = sqlIdentifier.names.asList().get(1); - String fieldKey = tableName + "_" + tableField; + String fieldKey = tableName + "." + tableField; if(tableName.equalsIgnoreCase(oldTbName)){ - String newFieldName = fieldReplaceRef.get(fieldKey) == null ? tableField : fieldReplaceRef.get(fieldKey); + /* + * ****Before replace:***** + * fieldKey: b.department + * fieldReplaceRef : b.department -> a_b_0.department0 + * oldFieldRef: a_b_0.department0 + * oldTbName: b + * oldFieldName: department + * ****After replace:***** + * newTbName: a_b_0 + * newFieldName: department0 + */ + String oldFieldRef = fieldReplaceRef.get(fieldKey); + String newFieldName = (oldFieldRef != null && !StringUtils.substringAfter(oldFieldRef, ".").isEmpty()) ? + StringUtils.substringAfter(oldFieldRef, ".") : tableField; + SqlIdentifier newField = ((SqlIdentifier)selectNode).setName(0, newTbName); newField = newField.setName(1, newFieldName); ((SqlIdentifier)selectNode).assignNamesFrom(newField); From c36e131dc912ecedb4f099f0ba54e0671b81bd7a Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 19 Jan 2021 14:03:55 +0800 Subject: [PATCH 368/378] =?UTF-8?q?[hotfix-34751][core]=20udf=20=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=E4=BD=BF=E7=94=A8=E5=BD=93=E5=89=8D=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?classloader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/classloader/ClassLoaderManager.java | 14 +++++++++++++- .../flink/sql/exec/ExecuteProcessHelper.java | 10 +--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java index 2e62e11ab..bd805007e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java @@ -20,10 +20,12 @@ import com.dtstack.flink.sql.util.PluginUtil; import com.dtstack.flink.sql.util.ReflectionUtils; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileInputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; @@ -73,7 +75,17 @@ private static DtClassLoader retrieveClassLoad(String pluginJarPath) { private static DtClassLoader retrieveClassLoad(List jarUrls) { jarUrls.sort(Comparator.comparing(URL::toString)); - String jarUrlkey = StringUtils.join(jarUrls, "_"); + + List jarMd5s = new ArrayList<>(jarUrls.size()); + for (URL jarUrl : jarUrls) { + try (FileInputStream inputStream = new FileInputStream(jarUrl.getPath())){ + String jarMd5 = DigestUtils.md5Hex(inputStream); + jarMd5s.add(jarMd5); + } catch (Exception e) { + throw new RuntimeException("Exceptions appears when read file:" + e); + } + } + String jarUrlkey = StringUtils.join(jarMd5s, "_"); return pluginClassLoader.computeIfAbsent(jarUrlkey, k -> { try { URL[] urls = jarUrls.toArray(new URL[jarUrls.size()]); diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java index d3a6f78be..97c40f93b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java +++ b/core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java @@ -248,18 +248,10 @@ private static void sqlTranslation(String localSqlPluginPath, public static void registerUserDefinedFunction(SqlTree sqlTree, List jarUrlList, TableEnvironment tableEnv, boolean isGetPlan) throws IllegalAccessException, InvocationTargetException { // udf和tableEnv须由同一个类加载器加载 - ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); - URLClassLoader classLoader = null; + URLClassLoader classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) currentClassLoader); List funcList = sqlTree.getFunctionList(); for (CreateFuncParser.SqlParserResult funcInfo : funcList) { - if (isGetPlan) { - classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) currentClassLoader); - } - //classloader - if (classLoader == null) { - classLoader = ClassLoaderManager.loadExtraJar(jarUrlList, (URLClassLoader) levelClassLoader); - } FunctionManager.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); } } From 9e89588b9a081c053eb24412f04041a3434dad23 Mon Sep 17 00:00:00 2001 From: wuren Date: Thu, 4 Feb 2021 17:17:59 +0800 Subject: [PATCH 369/378] [fix-35290][core] fix watermark metrics throw a warning log --- .../flink/sql/watermarker/AbstractCustomerWaterMarker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java index d75d26a61..9991c7021 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbstractCustomerWaterMarker.java @@ -92,7 +92,6 @@ public IterationRuntimeContext getIterationRuntimeContext() { public void setRuntimeContext(RuntimeContext t) { this.runtimeContext = t; eventDelayGauge = new EventDelayGauge(); - t.getMetricGroup().getAllVariables().put("", fromSourceTag); t.getMetricGroup().gauge(MetricConstant.DT_EVENT_DELAY_GAUGE, eventDelayGauge); } From d0b8a1d7afd9d81e407962423bcc13803020cc1f Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 4 Feb 2021 20:51:44 +0800 Subject: [PATCH 370/378] [feat-34267][redis] redis sink and side support ipv6. --- redis5/pom.xml | 13 ++ redis5/redis5-side/pom.xml | 14 --- .../flink/sql/side/redis/RedisAllReqRow.java | 114 ++++++++++-------- .../sql/side/redis/RedisAsyncReqRow.java | 31 +++-- redis5/redis5-side/redis-side-core/pom.xml | 11 +- .../flink/sql/side/redis/enums/RedisType.java | 3 +- .../sql/side/redis/table/RedisSideReqRow.java | 4 +- redis5/redis5-sink/pom.xml | 14 --- .../sql/sink/redis/RedisOutputFormat.java | 64 ++++++---- 9 files changed, 137 insertions(+), 131 deletions(-) diff --git a/redis5/pom.xml b/redis5/pom.xml index 894f786ef..d8e63bef1 100644 --- a/redis5/pom.xml +++ b/redis5/pom.xml @@ -16,5 +16,18 @@ redis5-side + + + com.dtstack.flink + sql.core + 1.0-SNAPSHOT + provided + + + redis.clients + jedis + 2.9.0 + + \ No newline at end of file diff --git a/redis5/redis5-side/pom.xml b/redis5/redis5-side/pom.xml index c6623b9d2..10462574c 100644 --- a/redis5/redis5-side/pom.xml +++ b/redis5/redis5-side/pom.xml @@ -11,20 +11,6 @@ 4.0.0 sql.side.redis redis-side - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - redis.clients - jedis - 2.8.0 - - - redis-side-core diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java index 8e6a08791..2e3c0887b 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllReqRow.java @@ -26,6 +26,7 @@ import com.dtstack.flink.sql.side.redis.table.RedisSideReqRow; import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import com.esotericsoftware.minlog.Log; +import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -35,7 +36,6 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; -import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.HostAndPort; @@ -48,14 +48,18 @@ import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; - +import java.util.Arrays; import java.util.Calendar; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * @author yanxi */ @@ -63,6 +67,8 @@ public class RedisAllReqRow extends BaseAllReqRow { private static final long serialVersionUID = 7578879189085344807L; + private static final Pattern HOST_PORT_PATTERN = Pattern.compile("(?(.*)):(?\\d+)*"); + private static final Logger LOG = LoggerFactory.getLogger(RedisAllReqRow.class); private static final int CONN_RETRY_NUM = 3; @@ -73,9 +79,9 @@ public class RedisAllReqRow extends BaseAllReqRow { private RedisSideTableInfo tableInfo; - private AtomicReference>> cacheRef = new AtomicReference<>(); + private final AtomicReference>> cacheRef = new AtomicReference<>(); - private RedisSideReqRow redisSideReqRow; + private final RedisSideReqRow redisSideReqRow; public RedisAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -115,8 +121,8 @@ public void flatMap(CRow input, Collector out) throws Exception { for (int i = 0; i < sideInfo.getEqualValIndex().size(); i++) { Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.row().getField(conValIndex); - if(equalObj == null){ - if(sideInfo.getJoinType() == JoinType.LEFT){ + if (equalObj == null) { + if (sideInfo.getJoinType() == JoinType.LEFT) { Row data = fillData(input.row(), null); out.collect(new CRow(data, input.change())); } @@ -125,13 +131,13 @@ public void flatMap(CRow input, Collector out) throws Exception { inputParams.put(sideInfo.getEqualFieldList().get(i), equalObj.toString()); } String key = buildCacheKey(inputParams); - if(StringUtils.isBlank(key)){ + if (StringUtils.isBlank(key)) { return; } Map cacheMap = cacheRef.get().get(key); - if(MapUtils.isEmpty(cacheMap)){ - if(sideInfo.getJoinType() != JoinType.LEFT){ + if (MapUtils.isEmpty(cacheMap)) { + if (sideInfo.getJoinType() != JoinType.LEFT) { return; } Row data = fillData(input.row(), null); @@ -146,8 +152,8 @@ public void flatMap(CRow input, Collector out) throws Exception { private String buildCacheKey(Map refData) { StringBuilder keyBuilder = new StringBuilder(tableInfo.getTableName()); List primaryKeys = tableInfo.getPrimaryKeys(); - for(String primaryKey : primaryKeys){ - if(!refData.containsKey(primaryKey)){ + for (String primaryKey : primaryKeys) { + if (!refData.containsKey(primaryKey)) { return null; } keyBuilder.append("_").append(refData.get(primaryKey)); @@ -160,7 +166,7 @@ private void loadData(Map> tmpCache) throws SQLExcep JedisCommands jedis = null; try { StringBuilder keyPattern = new StringBuilder(tableInfo.getTableName()); - for (String key : tableInfo.getPrimaryKeys()) { + for (int i = 0; i < tableInfo.getPrimaryKeys().size(); i++) { keyPattern.append("_").append("*"); } jedis = getJedisWithRetry(CONN_RETRY_NUM); @@ -195,41 +201,54 @@ private JedisCommands getJedis(RedisSideTableInfo tableInfo) { String url = tableInfo.getUrl(); String password = tableInfo.getPassword(); String database = tableInfo.getDatabase() == null ? "0" : tableInfo.getDatabase(); + String masterName = tableInfo.getMasterName(); int timeout = tableInfo.getTimeout(); - if (timeout == 0){ - timeout = 1000; + if (timeout == 0) { + timeout = 10000; } String[] nodes = url.split(","); - String[] firstIpPort = nodes[0].split(":"); - String firstIp = firstIpPort[0]; - String firstPort = firstIpPort[1]; - Set addresses = new HashSet<>(); - Set ipPorts = new HashSet<>(); - for (String ipPort : nodes) { - ipPorts.add(ipPort); - String[] ipPortPair = ipPort.split(":"); - addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); - } - if (timeout == 0){ - timeout = 1000; - } JedisCommands jedis = null; GenericObjectPoolConfig poolConfig = setPoolConfig(tableInfo.getMaxTotal(), tableInfo.getMaxIdle(), tableInfo.getMinIdle()); - switch (RedisType.parse(tableInfo.getRedisType())){ - //单机 + switch (RedisType.parse(tableInfo.getRedisType())) { case STANDALONE: - pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); - jedis = pool.getResource(); + String firstIp = null; + String firstPort = null; + Matcher standalone = HOST_PORT_PATTERN.matcher(nodes[0]); + if (standalone.find()) { + firstIp = standalone.group("host").trim(); + firstPort = standalone.group("port").trim(); + } + if (Objects.nonNull(firstIp)) { + pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); + jedis = pool.getResource(); + } else { + throw new IllegalArgumentException( + String.format("redis url error. current url [%s]", nodes[0])); + } break; - //哨兵 case SENTINEL: - jedisSentinelPool = new JedisSentinelPool(tableInfo.getMasterName(), ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); + Set ipPorts = new HashSet<>(Arrays.asList(nodes)); + jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; - //集群 case CLUSTER: - jedis = new JedisCluster(addresses, timeout, timeout,1, poolConfig); + Set addresses = new HashSet<>(); + // 对ipv6 支持 + for (String node : nodes) { + Matcher matcher = HOST_PORT_PATTERN.matcher(node); + if (matcher.find()) { + String host = matcher.group("host").trim(); + String portStr = matcher.group("port").trim(); + if (org.apache.commons.lang3.StringUtils.isNotBlank(host) && org.apache.commons.lang3.StringUtils.isNotBlank(portStr)) { + // 转化为int格式的端口 + int port = Integer.parseInt(portStr); + addresses.add(new HostAndPort(host, port)); + } + } + } + jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); + break; default: break; } @@ -257,35 +276,32 @@ private JedisCommands getJedisWithRetry(int retryNum) { return null; } - private Set getRedisKeys(RedisType redisType, JedisCommands jedis, String keyPattern){ - if(!redisType.equals(RedisType.CLUSTER)){ + private Set getRedisKeys(RedisType redisType, JedisCommands jedis, String keyPattern) { + if (!redisType.equals(RedisType.CLUSTER)) { return ((Jedis) jedis).keys(keyPattern); } Set keys = new TreeSet<>(); - Map clusterNodes = ((JedisCluster)jedis).getClusterNodes(); - for(String k : clusterNodes.keySet()){ + Map clusterNodes = ((JedisCluster) jedis).getClusterNodes(); + for (String k : clusterNodes.keySet()) { JedisPool jp = clusterNodes.get(k); - Jedis connection = jp.getResource(); - try { + try (Jedis connection = jp.getResource()) { keys.addAll(connection.keys(keyPattern)); - } catch (Exception e){ - LOG.error("Getting keys error: {}", e); - } finally { - connection.close(); + } catch (Exception e) { + LOG.error("Getting keys error.", e); } } return keys; } - private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, String minIdle){ + private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, String minIdle) { GenericObjectPoolConfig config = new GenericObjectPoolConfig(); - if (maxTotal != null){ + if (maxTotal != null) { config.setMaxTotal(Integer.parseInt(maxTotal)); } - if (maxIdle != null){ + if (maxIdle != null) { config.setMaxIdle(Integer.parseInt(maxIdle)); } - if (minIdle != null){ + if (minIdle != null) { config.setMinIdle(Integer.parseInt(minIdle)); } return config; diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 954469049..534b3d291 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -73,7 +73,7 @@ public class RedisAsyncReqRow extends BaseAsyncReqRow { private RedisSideTableInfo redisSideTableInfo; - private RedisSideReqRow redisSideReqRow; + private final RedisSideReqRow redisSideReqRow; public RedisAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, AbstractSideTableInfo sideTableInfo) { super(new RedisAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); @@ -98,7 +98,7 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ case STANDALONE: RedisURI redisURI = RedisURI.create("redis://" + url); redisURI.setPassword(password); - redisURI.setDatabase(Integer.valueOf(database)); + redisURI.setDatabase(Integer.parseInt(database)); redisClient = RedisClient.create(redisURI); connection = redisClient.connect(); async = connection.async(); @@ -106,7 +106,7 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ case SENTINEL: RedisURI redisSentinelURI = RedisURI.create("redis-sentinel://" + url); redisSentinelURI.setPassword(password); - redisSentinelURI.setDatabase(Integer.valueOf(database)); + redisSentinelURI.setDatabase(Integer.parseInt(database)); redisSentinelURI.setSentinelMasterId(redisSideTableInfo.getMasterName()); redisClient = RedisClient.create(redisSentinelURI); connection = redisClient.connect(); @@ -135,21 +135,18 @@ public void handleAsyncInvoke(Map inputParams, CRow input, Resul return; } RedisFuture> future = ((RedisHashAsyncCommands) async).hgetall(key); - future.thenAccept(new Consumer>() { - @Override - public void accept(Map values) { - if (MapUtils.isNotEmpty(values)) { - try { - Row row = fillData(input.row(), values); - dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, values)); - resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); - } catch (Exception e) { - dealFillDataError(input, resultFuture, e); - } - } else { - dealMissKey(input, resultFuture); - dealCacheData(key,CacheMissVal.getMissKeyObj()); + future.thenAccept(values -> { + if (MapUtils.isNotEmpty(values)) { + try { + Row row = fillData(input.row(), values); + dealCacheData(key,CacheObj.buildCacheObj(ECacheContentType.SingleLine, values)); + resultFuture.complete(Collections.singleton(new CRow(row, input.change()))); + } catch (Exception e) { + dealFillDataError(input, resultFuture, e); } + } else { + dealMissKey(input, resultFuture); + dealCacheData(key,CacheMissVal.getMissKeyObj()); } }); } diff --git a/redis5/redis5-side/redis-side-core/pom.xml b/redis5/redis5-side/redis-side-core/pom.xml index 7fb45a291..88e0a00dd 100644 --- a/redis5/redis5-side/redis-side-core/pom.xml +++ b/redis5/redis5-side/redis-side-core/pom.xml @@ -12,15 +12,6 @@ sql.side.redis.core - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - jar - + jar \ No newline at end of file diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java index 048c32b08..614447a5d 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/enums/RedisType.java @@ -28,6 +28,7 @@ public static RedisType parse(int redisType){ return type; } } - throw new RuntimeException("unsupport redis type["+ redisType + "]"); + throw new IllegalArgumentException( + "unsupported redis type["+ redisType + "]"); } } diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 634cc66fa..af1a1f2e4 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -43,7 +43,7 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); - private BaseSideInfo sideInfo; + private final BaseSideInfo sideInfo; public RedisSideReqRow(BaseSideInfo sideInfo){ this.sideInfo = sideInfo; @@ -102,7 +102,7 @@ public void setRowField(Row row, Integer index, BaseSideInfo sideInfo, String va row.setField(index, Float.valueOf(value)); break; case "java.math.BigDecimal": - row.setField(index, BigDecimal.valueOf(Long.valueOf(value))); + row.setField(index, BigDecimal.valueOf(Long.parseLong(value))); break; case "java.sql.Timestamp": row.setField(index, Timestamp.valueOf(value)); diff --git a/redis5/redis5-sink/pom.xml b/redis5/redis5-sink/pom.xml index be6a445ed..4fe35fb09 100644 --- a/redis5/redis5-sink/pom.xml +++ b/redis5/redis5-sink/pom.xml @@ -16,20 +16,6 @@ redis-sink http://maven.apache.org - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - redis.clients - jedis - 2.9.0 - - - diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index ab97cf60e..d86f90685 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -38,13 +38,17 @@ import java.io.Closeable; import java.io.IOException; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author yanxi */ -public class RedisOutputFormat extends AbstractDtRichOutputFormat { +public class RedisOutputFormat extends AbstractDtRichOutputFormat> { private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); + private static final Pattern HOST_PORT_PATTERN = Pattern.compile("(?(.*)):(?\\d+)*"); + private String url; private String database = "0"; @@ -77,8 +81,6 @@ public class RedisOutputFormat extends AbstractDtRichOutputFormat { private JedisSentinelPool jedisSentinelPool; - private GenericObjectPoolConfig poolConfig; - private RedisOutputFormat(){ } @Override @@ -107,44 +109,60 @@ private GenericObjectPoolConfig setPoolConfig(String maxTotal, String maxIdle, S } private void establishConnection() { - poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); + GenericObjectPoolConfig poolConfig = setPoolConfig(maxTotal, maxIdle, minIdle); String[] nodes = StringUtils.split(url, ","); - String[] firstIpPort = StringUtils.split(nodes[0], ":"); - String firstIp = firstIpPort[0]; - String firstPort = firstIpPort[1]; - Set addresses = new HashSet<>(); - Set ipPorts = new HashSet<>(); - for (String ipPort : nodes) { - ipPorts.add(ipPort); - String[] ipPortPair = StringUtils.split(ipPort, ":"); - addresses.add(new HostAndPort(ipPortPair[0].trim(), Integer.valueOf(ipPortPair[1].trim()))); - } switch (RedisType.parse(redisType)){ case STANDALONE: - pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); - jedis = pool.getResource(); + String firstIp = null; + String firstPort = null; + Matcher standalone = HOST_PORT_PATTERN.matcher(nodes[0]); + if (standalone.find()) { + firstIp = standalone.group("host").trim(); + firstPort = standalone.group("port").trim(); + } + if (Objects.nonNull(firstIp)) { + pool = new JedisPool(poolConfig, firstIp, Integer.parseInt(firstPort), timeout, password, Integer.parseInt(database)); + jedis = pool.getResource(); + } else { + throw new IllegalArgumentException( + String.format("redis url error. current url [%s]", nodes[0])); + } break; case SENTINEL: + Set ipPorts = new HashSet<>(Arrays.asList(nodes)); jedisSentinelPool = new JedisSentinelPool(masterName, ipPorts, poolConfig, timeout, password, Integer.parseInt(database)); jedis = jedisSentinelPool.getResource(); break; case CLUSTER: + Set addresses = new HashSet<>(); + // 对ipv6 支持 + for (String node : nodes) { + Matcher matcher = HOST_PORT_PATTERN.matcher(node); + if (matcher.find()) { + String host = matcher.group("host").trim(); + String portStr = matcher.group("port").trim(); + if (StringUtils.isNotBlank(host) && StringUtils.isNotBlank(portStr)) { + // 转化为int格式的端口 + int port = Integer.parseInt(portStr); + addresses.add(new HostAndPort(host, port)); + } + } + } jedis = new JedisCluster(addresses, timeout, timeout, 10, password, poolConfig); break; default: - throw new RuntimeException("unsupport redis type[ " + redisType + "]"); + throw new IllegalArgumentException("unsupported redis type[ " + redisType + "]"); } } @Override - public void writeRecord(Tuple2 record) throws IOException { - Tuple2 tupleTrans = record; - Boolean retract = tupleTrans.getField(0); + public void writeRecord(Tuple2 record) throws IOException { + Boolean retract = record.getField(0); if (!retract) { return; } - Row row = tupleTrans.getField(1); + Row row = record.getField(1); if (row.getArity() != fieldNames.length) { return; } @@ -153,9 +171,7 @@ public void writeRecord(Tuple2 record) throws IOException { refData.put(fieldNames[i], row.getField(i)); } String redisKey = buildCacheKey(refData); - refData.entrySet().forEach(e ->{ - jedis.hset(redisKey, e.getKey(), String.valueOf(e.getValue())); - }); + refData.forEach((key, value) -> jedis.hset(redisKey, key, String.valueOf(value))); if (outRecords.getCount() % ROW_PRINT_FREQUENCY == 0){ LOG.info(record.toString()); From 28fc9264413ee876ea4c6747ddf2f2bff9b9a611 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 4 Feb 2021 21:23:10 +0800 Subject: [PATCH 371/378] [feat-34267][redis] fix multi-primary-key with whitespace throw null when buildCache key. multi-primary-key like 'PRIMARY KEY(id, name, ....)' --- .../flink/sql/table/AbstractTableParser.java | 26 +++++++++++-------- .../sql/sink/redis/table/RedisSinkParser.java | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java index f0898cfd6..3df278d5e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java @@ -26,10 +26,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.tuple.Tuple2; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Reason: @@ -44,16 +46,16 @@ public abstract class AbstractTableParser { private static final String NEST_JSON_FIELD_KEY = "nestFieldKey"; private static final String CHAR_TYPE_NO_LENGTH = "CHAR"; - private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)"); - private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); - private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); - private static Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); + private static final Pattern primaryKeyPattern = Pattern.compile("(?i)(^\\s*)PRIMARY\\s+KEY\\s*\\((.*)\\)"); + private static final Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+?)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static final Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$"); + private static final Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$"); + private static final Pattern typePattern = Pattern.compile("(\\S+)\\s+(\\w+.*)"); - private Map patternMap = Maps.newHashMap(); + private final Map patternMap = Maps.newHashMap(); - private Map handlerMap = Maps.newHashMap(); + private final Map handlerMap = Maps.newHashMap(); public AbstractTableParser() { addParserHandler(PRIMARY_KEY, primaryKeyPattern, this::dealPrimaryKey); @@ -112,7 +114,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) { if (matcher.find()) { fieldClass = dbTypeConvertToJavaType(CHAR_TYPE_NO_LENGTH); fieldExtraInfo = new AbstractTableInfo.FieldExtraInfo(); - fieldExtraInfo.setLength(Integer.valueOf(matcher.group(1))); + fieldExtraInfo.setLength(Integer.parseInt(matcher.group(1))); } else { fieldClass = dbTypeConvertToJavaType(fieldType); } @@ -140,9 +142,11 @@ private Tuple2 extractType(String fieldRow, String tableName) { } public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) { - String primaryFields = matcher.group(1).trim(); - String[] splitArray = primaryFields.split(","); - List primaryKeys = Lists.newArrayList(splitArray); + String primaryFields = matcher.group(2).trim(); + List primaryKeys = Arrays + .stream(primaryFields.split(",")) + .map(String::trim) + .collect(Collectors.toList()); tableInfo.setPrimaryKeys(primaryKeys); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 8961f7da9..44d127884 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -51,7 +51,7 @@ public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = Lists.newArrayList(); + List primaryKeysList; primaryKeysList = Arrays.asList(StringUtils.split(primaryKeysStr, ",")); redisTableInfo.setPrimaryKeys(primaryKeysList); } From 3b598af38c5b3ecb9d021c597d554b8ca4e4e093 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Sun, 7 Feb 2021 19:57:13 +0800 Subject: [PATCH 372/378] [feat-35355][redis] fix redis async side plugin connect error on sentinel mode. --- .../sql/side/redis/RedisAsyncReqRow.java | 68 ++++++++++++++----- .../sql/side/redis/table/RedisSideReqRow.java | 3 + 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 954469049..3dd39e759 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -18,20 +18,9 @@ package com.dtstack.flink.sql.side.redis; +import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.AbstractSideTableInfo; import com.dtstack.flink.sql.side.BaseAsyncReqRow; -import io.lettuce.core.KeyValue; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.api.async.RedisStringAsyncCommands; -import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.table.runtime.types.CRow; -import org.apache.flink.types.Row; - -import com.dtstack.flink.sql.enums.ECacheContentType; import com.dtstack.flink.sql.side.CacheMissVal; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -41,6 +30,7 @@ import com.dtstack.flink.sql.side.redis.table.RedisSideTableInfo; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisFuture; +import io.lettuce.core.RedisURI; import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.async.RedisHashAsyncCommands; import io.lettuce.core.api.async.RedisKeyAsyncCommands; @@ -48,12 +38,19 @@ import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import com.google.common.collect.Maps; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.types.Row; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; +import java.util.regex.Matcher; + /** * @author yanxi */ @@ -104,11 +101,34 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ async = connection.async(); break; case SENTINEL: - RedisURI redisSentinelURI = RedisURI.create("redis-sentinel://" + url); - redisSentinelURI.setPassword(password); - redisSentinelURI.setDatabase(Integer.valueOf(database)); - redisSentinelURI.setSentinelMasterId(redisSideTableInfo.getMasterName()); - redisClient = RedisClient.create(redisSentinelURI); + String[] urlSplit = StringUtils.split(url, ","); + RedisURI.Builder builder = null; + for (String item : urlSplit) { + Matcher mather = RedisSideReqRow.HOST_PORT_PATTERN.matcher(item); + if (mather.find()) { + builder = buildSentinelUri( + mather.group("host"), + mather.group("port"), + builder + ); + } else { + throw new IllegalArgumentException( + String.format("Illegal format with redis url [%s]", item) + ); + } + } + + if (Objects.nonNull(builder)) { + builder + .withPassword(tableInfo.getPassword()) + .withDatabase(Integer.parseInt(tableInfo.getDatabase())) + .withSentinelMasterId(tableInfo.getMasterName()); + } else { + throw new NullPointerException("build redis uri error!"); + } + + RedisURI uri = builder.build(); + redisClient = RedisClient.create(uri); connection = redisClient.connect(); async = connection.async(); break; @@ -123,6 +143,18 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } } + private RedisURI.Builder buildSentinelUri( + String host, + String port, + RedisURI.Builder builder) { + if (Objects.nonNull(builder)) { + builder.withSentinel(host, Integer.parseInt(port)); + } else { + builder = RedisURI.Builder.sentinel(host, Integer.parseInt(port)); + } + return builder; + } + @Override public Row fillData(Row input, Object sideInput) { return redisSideReqRow.fillData(input, sideInput); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 634cc66fa..a51be6543 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -29,6 +29,7 @@ import java.sql.Timestamp; import java.util.Map; import java.util.TimeZone; +import java.util.regex.Pattern; /** * redis fill row data @@ -41,6 +42,8 @@ public class RedisSideReqRow implements ISideReqRow, Serializable { private static final long serialVersionUID = 3751171828444748982L; + public static final Pattern HOST_PORT_PATTERN = Pattern.compile("(?(.*)):(?\\d+)*"); + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); private BaseSideInfo sideInfo; From c06b190bf6a261c009d7f9ca13cbc41bd73938e2 Mon Sep 17 00:00:00 2001 From: mowen Date: Fri, 5 Mar 2021 11:18:57 +0800 Subject: [PATCH 373/378] [hotfix-35517][elasticsearch5&6][sink]log framework has been replaced from logback to log4j. --- elasticsearch5/elasticsearch5-sink/pom.xml | 19 +++++++------------ elasticsearch6/elasticsearch6-sink/pom.xml | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/elasticsearch5/elasticsearch5-sink/pom.xml b/elasticsearch5/elasticsearch5-sink/pom.xml index 515410c6e..0ad18338a 100644 --- a/elasticsearch5/elasticsearch5-sink/pom.xml +++ b/elasticsearch5/elasticsearch5-sink/pom.xml @@ -19,24 +19,19 @@ 5.3.3 - - ch.qos.logback - logback-core - 1.1.7 - - - - ch.qos.logback - logback-classic - 1.1.7 - - org.elasticsearch.client x-pack-transport 5.3.3 + + + org.slf4j + slf4j-log4j12 + 1.6.1 + + org.apache.logging.log4j log4j-to-slf4j diff --git a/elasticsearch6/elasticsearch6-sink/pom.xml b/elasticsearch6/elasticsearch6-sink/pom.xml index c1e755b2d..8caa7eeb8 100644 --- a/elasticsearch6/elasticsearch6-sink/pom.xml +++ b/elasticsearch6/elasticsearch6-sink/pom.xml @@ -30,6 +30,25 @@ ${elasticsearch.version} + + log4j + log4j + 1.2.17 + + + + slf4j-api + org.slf4j + 1.7.15 + + + + + org.slf4j + slf4j-log4j12 + 1.6.1 + + org.apache.flink flink-connector-elasticsearch6_2.11 From 79692cc7f1e0b1cd283374073b9d2f4a424b72a9 Mon Sep 17 00:00:00 2001 From: tiezhu Date: Thu, 4 Mar 2021 19:56:12 +0800 Subject: [PATCH 374/378] =?UTF-8?q?[hotfix-35512][impala]=201.=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=89=B9=E9=87=8F=E5=86=99=E5=85=A5=E6=97=B6=E5=87=BA?= =?UTF-8?q?=E7=8E=B0=E5=BC=82=E5=B8=B8=EF=BC=8C=E5=B0=86=E6=89=B9=E9=87=8F?= =?UTF-8?q?sql=E8=BD=AC=E4=B8=BA=E5=8D=95=E6=9D=A1sql=E6=89=A7=E8=A1=8C=20?= =?UTF-8?q?2.fix=20inserting=20the=20value=20like=20'[a,b]'=20or=20'{a,=20?= =?UTF-8?q?b}'=20or=20'=20'=20error=20with=20impala=20sink.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/sql/exception/ExceptionTrace.java | 39 +++ .../dtstack/flink/sql/util/DtStringUtil.java | 22 +- .../sql/sink/impala/ImpalaOutputFormat.java | 263 +++++++++--------- 3 files changed, 183 insertions(+), 141 deletions(-) create mode 100644 core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java diff --git a/core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java b/core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java new file mode 100644 index 000000000..07db131d9 --- /dev/null +++ b/core/src/main/java/com/dtstack/flink/sql/exception/ExceptionTrace.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.exception; + +import java.util.Objects; + +/** + * @author tiezhu + * @date 2021/3/8 星期一 + * Company dtstack + */ +public class ExceptionTrace { + // 追溯当前异常的最原始异常信息 + public static String traceOriginalCause(Throwable e) { + String errorMsg; + if (Objects.nonNull(e.getCause())) { + errorMsg = traceOriginalCause(e.getCause()); + } else { + errorMsg = e.getMessage(); + } + return errorMsg; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 5af657512..dbf6d9e7b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -21,15 +21,17 @@ package com.dtstack.flink.sql.util; import com.dtstack.flink.sql.enums.ColumnType; -import org.apache.commons.lang3.StringUtils; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.Maps; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.sql.Timestamp; +import org.apache.commons.lang3.StringUtils; + import java.math.BigDecimal; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -204,7 +206,6 @@ public static String dealSqlComment(String sql) { return b.toString(); } - public static String col2string(Object column, String type) { String rowData = column.toString(); ColumnType columnType = ColumnType.valueOf(type.toUpperCase()); @@ -374,8 +375,7 @@ public static String getTableFullPath(String schema, String tableName) { return addQuoteForStr(tableName); } - String schemaAndTabName = addQuoteForStr(schema) + "." + addQuoteForStr(tableName); - return schemaAndTabName; + return addQuoteForStr(schema) + "." + addQuoteForStr(tableName); } /** @@ -413,4 +413,14 @@ public static String removeStartAndEndQuota(String str) { String removeStart = StringUtils.removeStart(str, "'"); return StringUtils.removeEnd(removeStart, "'"); } + + /** + * 判断当前对象是null 还是空 + * + * @param obj 需要判断的对象 + * @return 返回true 如果对象是空或者为null + */ + public static boolean isEmptyOrNull(Object obj) { + return Objects.isNull(obj) || obj.toString().isEmpty(); + } } diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 35b0c9a50..82122e00f 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -18,10 +18,12 @@ package com.dtstack.flink.sql.sink.impala; +import com.dtstack.flink.sql.exception.ExceptionTrace; import com.dtstack.flink.sql.factory.DTThreadFactory; import com.dtstack.flink.sql.outputformat.AbstractDtRichOutputFormat; import com.dtstack.flink.sql.sink.rdb.JDBCTypeConvertUtils; import com.dtstack.flink.sql.table.AbstractTableInfo; +import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.JDBCUtils; import com.dtstack.flink.sql.util.KrbUtils; import com.google.common.collect.Maps; @@ -41,7 +43,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -72,8 +73,6 @@ public class ImpalaOutputFormat extends AbstractDtRichOutputFormat cast('value' as string) cast(value as timestamp) -> cast('value' as timestamp) - private static final Pattern TYPE_PATTERN = Pattern.compile("cast\\((.*) as (.*)\\)"); //specific type which values need to be quoted private static final String[] NEED_QUOTE_TYPE = {"string", "timestamp", "varchar"}; @@ -178,27 +177,20 @@ private void init() throws SQLException { if (!storeType.equalsIgnoreCase(KUDU_TYPE)) { throw new IllegalArgumentException("update mode not support for non-kudu table!"); } - updateStatement = connection.prepareStatement(buildUpdateSql(schema, tableName, fieldNames, primaryKeys)); - return; + } else { + valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields); } - - valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields); } private void initScheduledTask(Long batchWaitInterval) { try { if (batchWaitInterval != 0) { this.scheduler = new ScheduledThreadPoolExecutor(1, - new DTThreadFactory("impala-upsert-output-format")); + new DTThreadFactory("impala-upsert-output-format")); this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> { synchronized (ImpalaOutputFormat.this) { - try { - flush(); - } catch (Exception e) { - LOG.error("Writing records to impala jdbc failed.", e); - throw new RuntimeException("Writing records to impala jdbc failed.", e); - } + flush(); } }, batchWaitInterval, batchWaitInterval, TimeUnit.MILLISECONDS); } @@ -237,16 +229,16 @@ private void openJdbc() { } } - private void flush() throws Exception { - if (batchCount > 0) { - if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { - executeUpdateBatch(); - } - if (!rowDataMap.isEmpty()) { - String templateSql = + private synchronized void flush() { + try { + if (batchCount > 0) { + if (updateMode.equalsIgnoreCase(UPDATE_MODE)) { + executeUpdateBatch(); + } + if (!rowDataMap.isEmpty()) { + String templateSql = "INSERT INTO tableName ${tableFieldsCondition} PARTITION ${partitionCondition} VALUES ${valuesCondition}"; - executeBatchSql( - statement, + executeBatchSql( templateSql, schema, tableName, @@ -255,12 +247,15 @@ private void flush() throws Exception { valueFieldNames, partitionFields, rowDataMap - ); - rowDataMap.clear(); + ); + rowDataMap.clear(); + } } + batchCount = 0; + } catch (Exception e) { + LOG.error("Writing records to impala jdbc failed.", e); + throw new RuntimeException("Writing records to impala jdbc failed.", e); } - batchCount = 0; - } /** @@ -273,10 +268,10 @@ private void executeUpdateBatch() throws SQLException { rows.forEach(row -> { try { JDBCTypeConvertUtils.setRecordToStatement( - updateStatement, - JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), - row, - primaryKeys.stream().mapToInt(fieldNames::indexOf).toArray() + updateStatement, + JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), + row, + primaryKeys.stream().mapToInt(fieldNames::indexOf).toArray() ); updateStatement.addBatch(); } catch (Exception e) { @@ -288,8 +283,8 @@ private void executeUpdateBatch() throws SQLException { rows.clear(); } catch (Exception e) { LOG.debug("impala jdbc execute batch error ", e); - connection.rollback(); - connection.commit(); + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); updateStatement.clearBatch(); executeUpdate(connection); } @@ -300,14 +295,10 @@ public void executeUpdate(Connection connection) { try { setRecordToStatement(updateStatement, JDBCTypeConvertUtils.getSqlTypeFromFieldType(fieldTypes), row); updateStatement.executeUpdate(); - connection.commit(); + JDBCUtils.commit(connection); } catch (Exception e) { - try { - connection.rollback(); - connection.commit(); - } catch (SQLException e1) { - throw new RuntimeException(e1); - } + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); if (metricOutputFormat.outDirtyRecords.getCount() % DIRTY_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { LOG.error("record insert failed ,this row is {}", row.toString()); LOG.error("", e); @@ -331,7 +322,7 @@ private void putRowIntoMap(Map> rowDataMap, Tuple2 rebuildFieldNameListAndTypeList(List fieldNames, List staticPartitionFields, List fieldTypes, String partitionFields) { - if (partitionFields.isEmpty()) { + if (partitionFields == null || partitionFields.isEmpty()) { return fieldNames; } @@ -354,40 +345,6 @@ private List rebuildFieldNameListAndTypeList(List fieldNames, Li return valueFields; } - /** - * Quote a specific type of value, like string, timestamp - * before: 1, cast(tiezhu as string), cast(2001-01-09 01:05:01 as timestamp), cast(123 as int) - * after: 1, cast('tiezhu' as string), cast('2001-01-09 01:05:01' as timestamp), cast(123 as int) - * if cast value is null, then cast(null as type) - * - * @param valueCondition original value condition - * @return quoted condition - */ - private String valueConditionAddQuotation(String valueCondition) { - String[] temps = valueCondition.split(","); - List replacedItem = new ArrayList<>(); - Arrays.stream(temps).forEach( - item -> { - Matcher matcher = TYPE_PATTERN.matcher(item); - while (matcher.find()) { - String value = matcher.group(1); - String type = matcher.group(2); - - for (String needQuoteType : NEED_QUOTE_TYPE) { - if (type.contains(needQuoteType)) { - if (!"null".equals(value)) { - item = item.replace(value, "'" + value + "'"); - } - } - } - } - replacedItem.add(item); - } - ); - - return "(" + String.join(", ", replacedItem) + ")"; - } - @Override public void writeRecord(Tuple2 record) throws IOException { try { @@ -422,7 +379,7 @@ public void writeRecord(Tuple2 record) throws IOException { for (int i = 0; i < fieldTypes.size(); i++) { rowValue.setField(i, valueMap.get(valueFieldNames.get(i))); } - rowTuple2.f1 = valueConditionAddQuotation(buildValuesCondition(fieldTypes, rowValue)); + rowTuple2.f1 = buildValuesCondition(fieldTypes, rowValue); putRowIntoMap(rowDataMap, rowTuple2); } @@ -446,11 +403,7 @@ public void close() throws IOException { } // 将还未执行的SQL flush if (batchCount > 0) { - try { - flush(); - } catch (Exception e) { - throw new RuntimeException("Writing records to impala failed.", e); - } + flush(); } // cancel scheduled task if (this.scheduledFuture != null) { @@ -484,7 +437,6 @@ public void close() throws IOException { * execute batch sql from row data map * sql like 'insert into tableName(f1, f2, f3) ${partitionCondition} values(v1, v2, v3), (v4, v5, v6).... * - * @param statement execute statement * @param tempSql template sql * @param storeType the store type of data * @param enablePartition enable partition or not @@ -492,8 +444,7 @@ public void close() throws IOException { * @param partitionFields partition fields * @param rowDataMap row data map */ - private void executeBatchSql(Statement statement, - String tempSql, + private void executeBatchSql(String tempSql, String schema, String tableName, String storeType, @@ -501,46 +452,83 @@ private void executeBatchSql(Statement statement, List fieldNames, String partitionFields, Map> rowDataMap) { - StringBuilder valuesCondition = new StringBuilder(); StringBuilder partitionCondition = new StringBuilder(); String tableFieldsCondition = buildTableFieldsCondition(fieldNames, partitionFields); - ArrayList rowData; + ArrayList rowData = new ArrayList<>(); String tableNameInfo = Objects.isNull(schema) ? - tableName : quoteIdentifier(schema) + "." + tableName; + tableName : quoteIdentifier(schema) + "." + tableName; tempSql = tempSql.replace("tableName", tableNameInfo); + boolean isPartitioned = storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition; - // kudu ${partitionCondition} is null - if (storeType.equalsIgnoreCase(KUDU_TYPE) || !enablePartition) { - try { - rowData = rowDataMap.get(NO_PARTITION); - rowData.forEach(row -> valuesCondition.append(row).append(", ")); - String executeSql = tempSql.replace(VALUES_CONDITION, valuesCondition.toString()) + try { + // kudu ${partitionCondition} is null + if (isPartitioned) { + tempSql = tempSql + .replace(PARTITION_CONDITION, partitionCondition.toString()) + .replace(PARTITION_CONSTANT, "") + .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); + rowData.addAll(rowDataMap.get(NO_PARTITION)); + String executeSql = tempSql.replace(VALUES_CONDITION, String.join(", ", rowData)); + statement.execute(executeSql); + rowData.clear(); + } else { + // partition sql + Set keySet = rowDataMap.keySet(); + for (String key : keySet) { + rowData.addAll(rowDataMap.get(key)); + partitionCondition.append(key); + tempSql = tempSql .replace(PARTITION_CONDITION, partitionCondition.toString()) - .replace(PARTITION_CONSTANT, "") .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition); - String substring = executeSql.substring(0, executeSql.length() - 2); - statement.execute(substring); - } catch (Exception e) { - throw new RuntimeException("execute impala SQL error!", e); + String executeSql = tempSql + .replace(VALUES_CONDITION, String.join(", ", rowData)); + statement.execute(executeSql); + partitionCondition.delete(0, partitionCondition.length()); + } } - return; + } catch (Exception e) { + if (e instanceof SQLException) { + dealBatchSqlError(rowData, connection, statement, tempSql); + } else { + throw new RuntimeException("Insert into impala error!", e); + } + } finally { + rowData.clear(); } + } - // partition sql - Set keySet = rowDataMap.keySet(); - String finalTempSql = tempSql; - for (String key : keySet) { + /** + * 当批量写入失败时，把批量的sql拆解为单条sql提交，对于单条写入的sql记做脏数据 + * + * @param rowData 批量的values + * @param connection 当前数据库connect + * @param statement 当前statement + * @param templateSql 模版sql，例如insert into tableName(f1, f2, f3) [partition] values $valueCondition + */ + private void dealBatchSqlError(List rowData, + Connection connection, + Statement statement, + String templateSql) { + String errorMsg = "Insert into impala error. \nCause: [%s]\nRow: [%s]"; + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); + for (String rowDatum : rowData) { + String executeSql = templateSql.replace(VALUES_CONDITION, rowDatum); try { - String executeSql = String.copyValueOf(finalTempSql.toCharArray()); - ArrayList valuesConditionList = rowDataMap.get(key); - partitionCondition.append(key); - executeSql = executeSql.replace(PARTITION_CONDITION, partitionCondition.toString()) - .replace(TABLE_FIELDS_CONDITION, tableFieldsCondition) - .replace(VALUES_CONDITION, String.join(", ", valuesConditionList)); statement.execute(executeSql); - partitionCondition.delete(0, partitionCondition.length()); - } catch (SQLException sqlException) { - throw new RuntimeException("execute impala SQL error! ", sqlException); + JDBCUtils.commit(connection); + } catch (SQLException e) { + JDBCUtils.rollBack(connection); + JDBCUtils.commit(connection); + if (metricOutputFormat.outDirtyRecords.getCount() % DIRTY_DATA_PRINT_FREQUENCY == 0 || LOG.isDebugEnabled()) { + LOG.error( + String.format( + errorMsg, + ExceptionTrace.traceOriginalCause(e), + rowDatum) + ); + } + metricOutputFormat.outDirtyRecords.inc(); } } } @@ -573,9 +561,9 @@ private String buildPartitionCondition(Map rowData, String parti */ private String buildTableFieldsCondition(List fieldNames, String partitionFields) { return "(" + fieldNames.stream() - .filter(f -> !partitionFields.contains(f)) - .map(this::quoteIdentifier) - .collect(Collectors.joining(", ")) + ")"; + .filter(f -> !partitionFields.contains(f)) + .map(this::quoteIdentifier) + .collect(Collectors.joining(", ")) + ")"; } /** @@ -583,22 +571,27 @@ private String buildTableFieldsCondition(List fieldNames, String partiti * replace ${valuesCondition} * * @param fieldTypes field types - * @return condition like '(?, ?, cast(? as string))' and '?' will be replaced with row data + * @return condition like '(?, ?, cast('?' as string))' and '?' will be replaced with row data */ private String buildValuesCondition(List fieldTypes, Row row) { String valuesCondition = fieldTypes.stream().map( - f -> { - for (String item : NEED_QUOTE_TYPE) { - if (f.toLowerCase().contains(item)) { - return String.format("cast(? as %s)", f.toLowerCase()); - } + f -> { + for (String item : NEED_QUOTE_TYPE) { + if (f.toLowerCase().contains(item)) { + return String.format("cast('?' as %s)", f.toLowerCase()); } - return "?"; - }).collect(Collectors.joining(", ")); + } + return "?"; + }).collect(Collectors.joining(", ")); for (int i = 0; i < row.getArity(); i++) { - valuesCondition = valuesCondition.replaceFirst("\\?", Objects.isNull(row.getField(i)) ? "null" : row.getField(i).toString()); + Object rowField = row.getField(i); + if (DtStringUtil.isEmptyOrNull(rowField)) { + valuesCondition = valuesCondition.replaceFirst("'\\?'", "null"); + } else { + valuesCondition = valuesCondition.replaceFirst("\\?", rowField.toString()); + } } - return valuesCondition; + return "(" + valuesCondition + ")"; } /** @@ -609,16 +602,16 @@ private String buildValuesCondition(List fieldTypes, Row row) { private String buildUpdateSql(String schema, String tableName, List fieldNames, List primaryKeys) { //跳过primary key字段 String setClause = fieldNames.stream() - .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) - .map(f -> quoteIdentifier(f) + "=?") - .collect(Collectors.joining(", ")); + .filter(f -> !CollectionUtils.isNotEmpty(primaryKeys) || !primaryKeys.contains(f)) + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(", ")); String conditionClause = primaryKeys.stream() - .map(f -> quoteIdentifier(f) + "=?") - .collect(Collectors.joining(" AND ")); + .map(f -> quoteIdentifier(f) + "=?") + .collect(Collectors.joining(" AND ")); return "UPDATE " + (Objects.isNull(schema) ? "" : quoteIdentifier(schema) + ".") - + quoteIdentifier(tableName) + " SET " + setClause + " WHERE " + conditionClause; + + quoteIdentifier(tableName) + " SET " + setClause + " WHERE " + conditionClause; } private String quoteIdentifier(String identifier) { @@ -664,7 +657,7 @@ public Builder setTableName(String tableName) { public Builder setPartitionFields(String partitionFields) { format.partitionFields = Objects.isNull(partitionFields) ? - "" : partitionFields; + "" : partitionFields; return this; } @@ -739,11 +732,11 @@ public ImpalaOutputFormat build() { if (format.authMech == EAuthMech.Kerberos.getType()) { checkNotNull(format.krb5confPath, - "When kerberos authentication is enabled, krb5confPath is required！"); + "When kerberos authentication is enabled, krb5confPath is required！"); checkNotNull(format.principal, - "When kerberos authentication is enabled, principal is required！"); + "When kerberos authentication is enabled, principal is required！"); checkNotNull(format.keytabPath, - "When kerberos authentication is enabled, keytabPath is required！"); + "When kerberos authentication is enabled, keytabPath is required！"); } if (format.authMech == EAuthMech.UserName.getType()) { From 758c4db4b62fe2c5630058496989a99fdbfd9a51 Mon Sep 17 00:00:00 2001 From: mowen Date: Wed, 10 Mar 2021 14:52:25 +0800 Subject: [PATCH 375/378] [hotfix-34789][elasticsearch5&6][sink]support timestamp datatype. --- .../java/com/dtstack/flink/sql/enums/ColumnType.java | 4 ++++ .../java/com/dtstack/flink/sql/util/DtStringUtil.java | 3 +++ .../sql/sink/elasticsearch/CustomerSinkFunc.java | 11 +++++++++++ .../sql/sink/elasticsearch/CustomerSinkFunc.java | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java index 7f3f0019c..e93c92f51 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java +++ b/core/src/main/java/com/dtstack/flink/sql/enums/ColumnType.java @@ -102,6 +102,10 @@ public enum ColumnType { * timestamp */ TIMESTAMP, + /** + * time eg: 23:59:59 + */ + TIME, /** * decimal */ diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index dbf6d9e7b..bfa0c7ca5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -243,6 +243,9 @@ public static String col2string(Object column, String type) { case DATE: result = DateUtil.dateToString((java.util.Date)column); break; + case TIME: + result = DateUtil.getTimeFromStr(String.valueOf(column)); + break; case TIMESTAMP: result = DateUtil.timestampToString((java.util.Date)column); break; diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java index 23213e721..9c68ffea7 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/CustomerSinkFunc.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.sink.elasticsearch; +import com.dtstack.flink.sql.util.DateUtil; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.java.tuple.Tuple2; @@ -32,6 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Date; +import java.sql.Timestamp; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -107,6 +110,14 @@ private IndexRequest createIndexRequest(Row element) { Map dataMap = EsUtil.rowToJsonMap(element,fieldNames,fieldTypes); int length = Math.min(element.getArity(), fieldNames.size()); for(int i=0; i Date: Mon, 22 Mar 2021 10:00:53 +0800 Subject: [PATCH 376/378] =?UTF-8?q?[hotfix-35753][impala]=20fix=20String.r?= =?UTF-8?q?eplaceFirst=E4=B8=AD=E7=9A=84=E5=9D=91=EF=BC=8CreplaceFirst("\\?= =?UTF-8?q?=3F",=20value)=EF=BC=8C=E5=A6=82=E6=9E=9Cvalue=E4=B8=AD$?= =?UTF-8?q?=E5=90=8E=E4=B8=8D=E6=98=AF=E6=95=B0=E5=AD=97=EF=BC=8C=E9=82=A3?= =?UTF-8?q?=E4=B9=88=E4=BC=9A=E5=87=BA=E7=8E=B0"illegal=20group=20referenc?= =?UTF-8?q?e=EF=BC=9Agroup=20index=20is=20missing"=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java index 82122e00f..8723254bf 100644 --- a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java +++ b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java @@ -588,7 +588,7 @@ private String buildValuesCondition(List fieldTypes, Row row) { if (DtStringUtil.isEmptyOrNull(rowField)) { valuesCondition = valuesCondition.replaceFirst("'\\?'", "null"); } else { - valuesCondition = valuesCondition.replaceFirst("\\?", rowField.toString()); + valuesCondition = valuesCondition.replaceFirst("\\?", Matcher.quoteReplacement(rowField.toString())); } } return "(" + valuesCondition + ")"; From 1297b4fa75c206c32ded2c8fa99df8d5ca03b7bb Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Tue, 13 Apr 2021 17:13:08 +0800 Subject: [PATCH 377/378] =?UTF-8?q?[hotfix-36198][redis]=20=E8=A7=A3?= =?UTF-8?q?=E5=86=B3redis=E5=BC=82=E6=AD=A5=E7=BB=B4=E8=A1=A8cluster?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E8=BF=9E=E6=8E=A5=E5=BC=82=E5=B8=B8=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hbase/hbase-side/hbase-all-side/pom.xml | 4 +++ hbase/pom.xml | 4 +++ .../sql/side/redis/RedisAsyncReqRow.java | 27 ++++++++++++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/pom.xml b/hbase/hbase-side/hbase-all-side/pom.xml index cdfc24b74..dcc5a1987 100644 --- a/hbase/hbase-side/hbase-all-side/pom.xml +++ b/hbase/hbase-side/hbase-all-side/pom.xml @@ -33,6 +33,10 @@ io.netty netty + + netty-all + io.netty + diff --git a/hbase/pom.xml b/hbase/pom.xml index 835457e29..9c82560b3 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -47,6 +47,10 @@ io.netty netty + + netty-all + io.netty + diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index 395aa50a5..5fc8f62b9 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -36,6 +36,7 @@ import io.lettuce.core.api.async.RedisKeyAsyncCommands; import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; +import io.lettuce.core.internal.HostAndPort; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -44,11 +45,11 @@ import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.types.Row; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Consumer; import java.util.regex.Matcher; /** @@ -133,9 +134,8 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ async = connection.async(); break; case CLUSTER: - RedisURI clusterURI = RedisURI.create("redis://" + url); - clusterURI.setPassword(password); - clusterClient = RedisClusterClient.create(clusterURI); + List clusterURIs = buildClusterURIs(url); + clusterClient = RedisClusterClient.create(clusterURIs); clusterConnection = clusterClient.connect(); async = clusterConnection.async(); default: @@ -143,6 +143,25 @@ private void buildRedisClient(RedisSideTableInfo tableInfo){ } } + private List buildClusterURIs(String url) { + String password = redisSideTableInfo.getPassword(); + String database = redisSideTableInfo.getDatabase(); + String[] addresses = StringUtils.split(url, ","); + List redisURIs = new ArrayList<>(addresses.length); + for (String addr : addresses){ + HostAndPort hostAndPort = HostAndPort.parse(addr); + RedisURI redisURI = RedisURI.create(hostAndPort.hostText, hostAndPort.port); + if (StringUtils.isNotEmpty(password)) { + redisURI.setPassword(password); + } + if (StringUtils.isNotEmpty(database)) { + redisURI.setDatabase(Integer.valueOf(database)); + } + redisURIs.add(redisURI); + } + return redisURIs; + } + private RedisURI.Builder buildSentinelUri( String host, String port, From 34e200b58fb8a89bedb2f3228f978fc123bdbf47 Mon Sep 17 00:00:00 2001 From: xiuzhu Date: Thu, 15 Apr 2021 18:51:19 +0800 Subject: [PATCH 378/378] [hotfix-36198] redis async side shade netty --- hbase/hbase-side/hbase-all-side/pom.xml | 4 ---- hbase/pom.xml | 4 ---- redis5/redis5-side/redis-async-side/pom.xml | 8 +++++++- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hbase/hbase-side/hbase-all-side/pom.xml b/hbase/hbase-side/hbase-all-side/pom.xml index dcc5a1987..cdfc24b74 100644 --- a/hbase/hbase-side/hbase-all-side/pom.xml +++ b/hbase/hbase-side/hbase-all-side/pom.xml @@ -33,10 +33,6 @@ io.netty netty - - netty-all - io.netty - diff --git a/hbase/pom.xml b/hbase/pom.xml index 9c82560b3..835457e29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -47,10 +47,6 @@ io.netty netty - - netty-all - io.netty - diff --git a/redis5/redis5-side/redis-async-side/pom.xml b/redis5/redis5-side/redis-async-side/pom.xml index 39a6280cc..666bace2b 100644 --- a/redis5/redis5-side/redis-async-side/pom.xml +++ b/redis5/redis5-side/redis-async-side/pom.xml @@ -35,7 +35,7 @@ org.apache.maven.plugins maven-shade-plugin - 1.4 + 3.0.0 package @@ -43,6 +43,12 @@ shade + + + io.netty + com.dtstack.flink.sql.side.redis.io.netty + +