Skip to content

Commit e07325b

Browse files
ayushbilalaebyhr
ayushbilala
authored and
ebyhr
committed
Make BigQuery views cache ttl configurable
1 parent 8f07f96 commit e07325b

4 files changed

Lines changed: 42 additions & 23 deletions

File tree

‎plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryClient.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import java.util.Optional;
4545
import java.util.Set;
4646
import java.util.concurrent.Callable;
47+
import java.util.concurrent.ExecutionException;
4748
import java.util.concurrent.TimeUnit;
4849
import java.util.function.Supplier;
4950

@@ -54,6 +55,7 @@
5455
import static com.google.common.collect.Iterables.getOnlyElement;
5556
import static com.google.common.collect.Streams.stream;
5657
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_AMBIGUOUS_OBJECT_NAME;
58+
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_VIEW_DESTINATION_TABLE_CREATION_FAILED;
5759
import static io.trino.plugin.bigquery.BigQueryUtil.convertToBigQueryException;
5860
import static java.lang.String.format;
5961
import static java.util.Locale.ENGLISH;
@@ -72,6 +74,7 @@ class BigQueryClient
7274
private final boolean caseInsensitiveNameMatching;
7375
private final Cache<String, Optional<RemoteDatabaseObject>> remoteDatasets;
7476
private final Cache<TableId, Optional<RemoteDatabaseObject>> remoteTables;
77+
private final Cache<String, TableInfo> destinationTableCache;
7578

7679
BigQueryClient(BigQuery bigQuery, BigQueryConfig config)
7780
{
@@ -85,6 +88,10 @@ class BigQueryClient
8588
.expireAfterWrite(caseInsensitiveNameMatchingCacheTtl.toMillis(), MILLISECONDS);
8689
this.remoteDatasets = remoteNamesCacheBuilder.build();
8790
this.remoteTables = remoteNamesCacheBuilder.build();
91+
this.destinationTableCache = CacheBuilder.newBuilder()
92+
.expireAfterWrite(config.getViewsCacheTtl().toMillis(), MILLISECONDS)
93+
.maximumSize(1000)
94+
.build();
8895
}
8996

9097
Optional<RemoteDatabaseObject> toRemoteDataset(String projectId, String datasetName)
@@ -182,6 +189,19 @@ TableInfo getTable(TableId remoteTableId)
182189
return bigQuery.getTable(remoteTableId);
183190
}
184191

192+
TableInfo getCachedTable(ReadSessionCreatorConfig config, TableId tableId, List<String> requiredColumns)
193+
{
194+
String query = selectSql(tableId, requiredColumns);
195+
log.debug("query is %s", query);
196+
try {
197+
return destinationTableCache.get(query,
198+
new DestinationTableBuilder(this, config, query, tableId));
199+
}
200+
catch (ExecutionException e) {
201+
throw new TrinoException(BIGQUERY_VIEW_DESTINATION_TABLE_CREATION_FAILED, "Error creating destination table", e);
202+
}
203+
}
204+
185205
String getProjectId()
186206
{
187207
return bigQuery.getOptions().getProjectId();

‎plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public class BigQueryConfig
4747
private int maxReadRowsRetries = DEFAULT_MAX_READ_ROWS_RETRIES;
4848
private boolean caseInsensitiveNameMatching;
4949
private Duration caseInsensitiveNameMatchingCacheTtl = new Duration(1, MINUTES);
50+
private Duration viewsCacheTtl = new Duration(15, MINUTES);
5051

5152
@AssertTrue(message = "Exactly one of 'bigquery.credentials-key' or 'bigquery.credentials-file' must be specified, or the default GoogleCredentials could be created")
5253
public boolean isCredentialsConfigurationValid()
@@ -219,6 +220,21 @@ public BigQueryConfig setCaseInsensitiveNameMatchingCacheTtl(Duration caseInsens
219220
return this;
220221
}
221222

223+
@NotNull
224+
@MinDuration("0m")
225+
public Duration getViewsCacheTtl()
226+
{
227+
return viewsCacheTtl;
228+
}
229+
230+
@Config("bigquery.views-cache-ttl")
231+
@ConfigDescription("Duration for which the results of querying a view will be cached")
232+
public BigQueryConfig setViewsCacheTtl(Duration viewsCacheTtl)
233+
{
234+
this.viewsCacheTtl = viewsCacheTtl;
235+
return this;
236+
}
237+
222238
ReadSessionCreatorConfig createReadSessionCreatorConfig()
223239
{
224240
return new ReadSessionCreatorConfig(

‎plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadSessionCreator.java‎

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,32 +20,18 @@
2020
import com.google.cloud.bigquery.storage.v1beta1.ReadOptions;
2121
import com.google.cloud.bigquery.storage.v1beta1.Storage;
2222
import com.google.cloud.bigquery.storage.v1beta1.TableReferenceProto;
23-
import com.google.common.cache.Cache;
24-
import com.google.common.cache.CacheBuilder;
25-
import io.airlift.log.Logger;
2623
import io.trino.spi.TrinoException;
2724

2825
import java.util.List;
2926
import java.util.Optional;
30-
import java.util.concurrent.ExecutionException;
31-
import java.util.concurrent.TimeUnit;
3227

33-
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_VIEW_DESTINATION_TABLE_CREATION_FAILED;
3428
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
3529
import static java.lang.String.format;
3630
import static java.util.stream.Collectors.toList;
3731

3832
// A helper class, also handles view materialization
3933
public class ReadSessionCreator
4034
{
41-
private static final Logger log = Logger.get(ReadSessionCreator.class);
42-
43-
private static final Cache<String, TableInfo> destinationTableCache =
44-
CacheBuilder.newBuilder()
45-
.expireAfterWrite(15, TimeUnit.MINUTES)
46-
.maximumSize(1000)
47-
.build();
48-
4935
private final ReadSessionCreatorConfig config;
5036
private final BigQueryClient bigQueryClient;
5137
private final BigQueryStorageClientFactory bigQueryStorageClientFactory;
@@ -118,14 +104,7 @@ private TableInfo getActualTable(
118104
BigQueryConfig.VIEWS_ENABLED));
119105
}
120106
// get it from the view
121-
String query = bigQueryClient.selectSql(table.getTableId(), requiredColumns);
122-
log.debug("query is %s", query);
123-
try {
124-
return destinationTableCache.get(query, new BigQueryClient.DestinationTableBuilder(bigQueryClient, config, query, table.getTableId()));
125-
}
126-
catch (ExecutionException e) {
127-
throw new TrinoException(BIGQUERY_VIEW_DESTINATION_TABLE_CREATION_FAILED, "Error creating destination table", e);
128-
}
107+
return bigQueryClient.getCachedTable(config, table.getTableId(), requiredColumns);
129108
}
130109
else {
131110
// not regular table or a view

‎plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ public void testDefaults()
4444
.setViewMaterializationDataset("vmdataset")
4545
.setMaxReadRowsRetries(10)
4646
.setCaseInsensitiveNameMatching(false)
47-
.setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, MINUTES));
47+
.setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, MINUTES))
48+
.setViewsCacheTtl(new Duration(15, MINUTES));
4849

4950
assertEquals(config.getCredentialsKey(), Optional.of("key"));
5051
assertEquals(config.getCredentialsFile(), Optional.of("cfile"));
@@ -56,6 +57,7 @@ public void testDefaults()
5657
assertEquals(config.getMaxReadRowsRetries(), 10);
5758
assertEquals(config.isCaseInsensitiveNameMatching(), false);
5859
assertEquals(config.getCaseInsensitiveNameMatchingCacheTtl(), new Duration(1, MINUTES));
60+
assertEquals(config.getViewsCacheTtl(), new Duration(15, MINUTES));
5961
}
6062

6163
@Test
@@ -72,6 +74,7 @@ public void testExplicitPropertyMappingsWithCredentialsKey()
7274
.put("bigquery.max-read-rows-retries", "10")
7375
.put("bigquery.case-insensitive-name-matching", "true")
7476
.put("bigquery.case-insensitive-name-matching.cache-ttl", "1s")
77+
.put("bigquery.views-cache-ttl", "1m")
7578
.build();
7679

7780
ConfigurationFactory configurationFactory = new ConfigurationFactory(properties);
@@ -87,6 +90,7 @@ public void testExplicitPropertyMappingsWithCredentialsKey()
8790
assertEquals(config.getMaxReadRowsRetries(), 10);
8891
assertEquals(config.isCaseInsensitiveNameMatching(), true);
8992
assertEquals(config.getCaseInsensitiveNameMatchingCacheTtl(), new Duration(1, SECONDS));
93+
assertEquals(config.getViewsCacheTtl(), new Duration(1, MINUTES));
9094
}
9195

9296
@Test

0 commit comments

Comments
 (0)