Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.Text;

/** {@link ResultSet} Implementation for Arrow datasource (Using Storage Read APIs) */
class BigQueryArrowResultSet extends BigQueryBaseResultSet {
Expand Down Expand Up @@ -345,6 +346,27 @@ private Object getObjectInternal(int columnIndex) throws SQLException {
return value;
}

@Override
public String getString(int columnIndex) throws SQLException {
checkClosed();
StandardSQLTypeName type = getStandardSQLTypeName(columnIndex);
if (type != StandardSQLTypeName.TIMESTAMP) {
return super.getString(columnIndex);
}
Object value = getObjectInternal(columnIndex);
if (value == null) {
return null;
}
if (value instanceof Text || value instanceof String) {
return BigQueryTemporalUtility.formatTimestampStringFromIso(
value.toString(), this.statement.isEnableTimestampPicos());
}
if (value instanceof Long) {
return BigQueryTemporalUtility.formatTimestampStringFromMicroseconds((Long) value);
}
return super.getString(columnIndex);
}

@Override
public Object getObject(int columnIndex) throws SQLException {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
int highThroughputMinTableSize;
int highThroughputActivationRatio;
boolean enableSession;
boolean enableTimestampPicos;
boolean enableProjectDiscovery;
private List<String> discoveredProjectsCache;
boolean unsupportedHTAPIFallback;
Expand Down Expand Up @@ -372,6 +373,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
this.sslTrustStoreProvider,
this.connectionClassName);
this.enableSession = ds.getEnableSession();
this.enableTimestampPicos = ds.getEnableTimestampPicos();
this.unsupportedHTAPIFallback = ds.getUnsupportedHTAPIFallback();
this.maxResults = ds.getMaxResults();
Map<String, String> queryPropertiesMap = ds.getQueryProperties();
Expand Down Expand Up @@ -707,6 +709,10 @@ boolean isSessionEnabled() {
return this.enableSession;
}

boolean isEnableTimestampPicos() {
return this.enableTimestampPicos;
}

boolean isUnsupportedHTAPIFallback() {
return this.unsupportedHTAPIFallback;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,8 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
static final String SSL_TRUST_STORE_TYPE_PROPERTY_NAME = "SSLTrustStoreType";
static final String SSL_TRUST_STORE_PROVIDER_PROPERTY_NAME = "SSLTrustStoreProvider";
static final int DEFAULT_REQUEST_GOOGLE_DRIVE_SCOPE_VALUE = 0;
static final String ENABLE_TIMESTAMP_PICOS_PROPERTY_NAME = "EnableTimestampPicos";
static final boolean DEFAULT_ENABLE_TIMESTAMP_PICOS_VALUE = false;
static final String MAX_BYTES_BILLED_PROPERTY_NAME = "MaximumBytesBilled";
static final Long DEFAULT_MAX_BYTES_BILLED_VALUE = 0L;
static final String LABELS_PROPERTY_NAME = "Labels";
Expand Down Expand Up @@ -310,6 +312,12 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
Collections.unmodifiableSet(
new HashSet<>(
Arrays.asList(
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_TIMESTAMP_PICOS_PROPERTY_NAME)
.setDescription(
"Enables or disables 12-digit picosecond precision for TIMESTAMP columns. Disabled by default.")
.setDefaultValue(String.valueOf(DEFAULT_ENABLE_TIMESTAMP_PICOS_VALUE))
Comment thread
keshavdandeva marked this conversation as resolved.
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(MAX_BYTES_BILLED_PROPERTY_NAME)
.setDescription(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ public String getString(int columnIndex) throws SQLException {
if (value.getAttribute() == Attribute.REPEATED || value.getAttribute() == Attribute.RECORD) {
return super.getString(columnIndex);
}
return BigQueryTemporalUtility.formatTimestampString(value.getStringValue());
return BigQueryTemporalUtility.formatTimestampString(
value.getStringValue(), this.statement.isEnableTimestampPicos());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,16 @@ Statement getStatement() {
return this.statement;
}

private boolean isTimestampPicosEnabled() {
return this.statement instanceof BigQueryStatement
&& ((BigQueryStatement) this.statement).isEnableTimestampPicos();
}

private boolean supportsPicoseconds(int sqlColumn) {
Long timestampPrecision = getField(sqlColumn).getTimestampPrecision();
return timestampPrecision != null && timestampPrecision > 6;
}

private Field getField(int sqlColumn) {
return this.schemaFieldList.get(sqlColumn - 1);
}
Expand Down Expand Up @@ -116,7 +126,10 @@ public int getColumnDisplaySize(int column) {
case Types.NUMERIC:
return 14;
case Types.TIMESTAMP:
return 16;
if (isTimestampPicosEnabled() && supportsPicoseconds(column)) {
return 32;
}
return 26;
default:
return DEFAULT_DISPLAY_SIZE;
}
Expand All @@ -139,6 +152,11 @@ public int getPrecision(int column) {
return precision.intValue();
}
StandardSQLTypeName type = getStandardSQLTypeName(column);
if (type == StandardSQLTypeName.TIMESTAMP
&& isTimestampPicosEnabled()
&& supportsPicoseconds(column)) {
return 32;
}
BigQueryJdbcTypeMappings.ColumnTypeInfo typeInfo =
BigQueryJdbcTypeMappings.STANDARD_TYPE_INFO.get(type);
if (typeInfo != null && typeInfo.columnSize != null) {
Expand All @@ -154,6 +172,11 @@ public int getScale(int column) {
return scale.intValue();
}
StandardSQLTypeName type = getStandardSQLTypeName(column);
if (type == StandardSQLTypeName.TIMESTAMP
&& isTimestampPicosEnabled()
&& supportsPicoseconds(column)) {
return 12;
}
BigQueryJdbcTypeMappings.ColumnTypeInfo typeInfo =
BigQueryJdbcTypeMappings.STANDARD_TYPE_INFO.get(type);
if (typeInfo != null && typeInfo.decimalDigits != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,14 @@
import com.google.cloud.bigquery.exception.BigQueryJdbcSqlSyntaxErrorException;
import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
import com.google.cloud.bigquery.storage.v1.ArrowSerializationOptions;
import com.google.cloud.bigquery.storage.v1.BigQueryReadClient;
import com.google.cloud.bigquery.storage.v1.CreateReadSessionRequest;
import com.google.cloud.bigquery.storage.v1.DataFormat;
import com.google.cloud.bigquery.storage.v1.ReadRowsRequest;
import com.google.cloud.bigquery.storage.v1.ReadRowsResponse;
import com.google.cloud.bigquery.storage.v1.ReadSession;
import com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.Uninterruptibles;
Expand Down Expand Up @@ -846,6 +848,15 @@ ResultSet processArrowResultSet(TableResult results, Job job) throws SQLExceptio
ReadSession.Builder sessionBuilder =
ReadSession.newBuilder().setTable(srcTable).setDataFormat(DataFormat.ARROW);

if (this.connection.isEnableTimestampPicos()) {
TableReadOptions.Builder tableReadOptionsBuilder = TableReadOptions.newBuilder();
tableReadOptionsBuilder
.getArrowSerializationOptionsBuilder()
.setPicosTimestampPrecision(
ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS);
sessionBuilder.setReadOptions(tableReadOptionsBuilder.build());
}

CreateReadSessionRequest.Builder builder =
CreateReadSessionRequest.newBuilder()
.setParent(parent)
Expand Down Expand Up @@ -1671,6 +1682,10 @@ public Connection getConnection() {
return this.connection;
}

boolean isEnableTimestampPicos() {
return this.connection.isEnableTimestampPicos();
}

public boolean hasMoreResults() {
if (this.parentJobId == null) {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,15 +164,6 @@ public static Instant parseEpochDecimalToInstant(String epochDecimal) {
return Instant.ofEpochSecond(seconds, nanos);
}

/**
* Formats a numeric epoch decimal string into standard SQL timestamp string format ("yyyy-MM-dd
* HH:mm:ss.ffffff"). Sub-microsecond precision is deterministically truncated (down) to prevent
* timestamp boundary rollovers.
*/
public static String formatTimestampString(String epochDecimal) {
return formatTimestampString(epochDecimal, false);
}

/**
* Formats a numeric epoch decimal string into standard SQL timestamp string format ("yyyy-MM-dd
* HH:mm:ss.ffffff[ffffff]"). Sub-microsecond / sub-picosecond precision is deterministically
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public class DataSource implements javax.sql.DataSource {
private Map<String, String> queryProperties;
private String logLevel;
private Boolean enableSession;
private Boolean enableTimestampPicos;
private String logPath;
private String gcpTelemetryProjectId;
private String gcpTelemetryCredentials;
Expand Down Expand Up @@ -149,6 +150,12 @@ public class DataSource implements javax.sql.DataSource {
.put(
BigQueryJdbcUrlUtility.GCP_TELEMETRY_CREDENTIALS_PROPERTY_NAME,
DataSource::setGcpTelemetryCredentials)
.put(
BigQueryJdbcUrlUtility.ENABLE_TIMESTAMP_PICOS_PROPERTY_NAME,
(ds, val) ->
ds.setEnableTimestampPicos(
BigQueryJdbcUrlUtility.convertIntToBoolean(
val, BigQueryJdbcUrlUtility.ENABLE_TIMESTAMP_PICOS_PROPERTY_NAME)))
.put(
BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME,
(ds, val) ->
Expand Down Expand Up @@ -925,6 +932,16 @@ public Boolean getUnsupportedHTAPIFallback() {
: BigQueryJdbcUrlUtility.DEFAULT_UNSUPPORTED_HTAPI_FALLBACK_VALUE;
}

public Boolean getEnableTimestampPicos() {
return enableTimestampPicos != null
? enableTimestampPicos
: BigQueryJdbcUrlUtility.DEFAULT_ENABLE_TIMESTAMP_PICOS_VALUE;
}

public void setEnableTimestampPicos(Boolean enableTimestampPicos) {
this.enableTimestampPicos = enableTimestampPicos;
}

public Boolean getEnableSession() {
return enableSession != null
? enableSession
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.sql.Time;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Period;
Expand Down Expand Up @@ -158,6 +159,26 @@ public void longToTimestamp() {
.isEqualTo(new Timestamp(1408452095220L));
}

@Test
public void textToTimestamp() {
Text textUtc = new Text("2026-04-08 10:00:00.123456789123 UTC");
Timestamp expected = Timestamp.from(Instant.parse("2026-04-08T10:00:00.123456789Z"));
assertThat(INSTANCE.coerceTo(Timestamp.class, textUtc)).isEqualTo(expected);

Text textIso = new Text("2026-04-08T10:00:00.123456789123Z");
assertThat(INSTANCE.coerceTo(Timestamp.class, textIso)).isEqualTo(expected);
}

@Test
public void stringToTimestamp() {
String strUtc = "2026-04-08 10:00:00.123456789123 UTC";
Timestamp expected = Timestamp.from(Instant.parse("2026-04-08T10:00:00.123456789Z"));
assertThat(INSTANCE.coerceTo(Timestamp.class, strUtc)).isEqualTo(expected);

String strIso = "2026-04-08T10:00:00.123456789123Z";
assertThat(INSTANCE.coerceTo(Timestamp.class, strIso)).isEqualTo(expected);
}

@Test
public void nullToTime() {
assertThat(INSTANCE.coerceTo(Time.class, null)).isNull();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.apache.arrow.vector.types.Types.MinorType.VARCHAR;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.Field.Mode;
Expand Down Expand Up @@ -437,5 +438,56 @@ private int resultSetRowCount(BigQueryArrowResultSet resultSet) throws SQLExcept
return rowCount;
}

// TODO: Unit Test for iteration and getters
@Test
public void testPicosecondTimestampArrowVector() throws Exception {
RootAllocator allocator = new RootAllocator();
VarCharVector timeStampPicosVector = new VarCharVector("timeStampField", allocator);
timeStampPicosVector.allocateNew(1);
timeStampPicosVector.set(0, new Text("2026-04-08T10:00:00.123456789123Z"));
timeStampPicosVector.setValueCount(1);

VectorSchemaRoot picosRoot = new VectorSchemaRoot(ImmutableList.of(timeStampPicosVector));
ArrowSchema arrowSchema =
ArrowSchema.newBuilder()
.setSerializedSchema(serializeSchema(picosRoot.getSchema()))
.build();
ArrowRecordBatch recordBatch =
ArrowRecordBatch.newBuilder()
.setSerializedRecordBatch(serializeVectorSchemaRoot(picosRoot))
.build();

BigQueryArrowBatchWrapper batchWrapper = BigQueryArrowBatchWrapper.of(recordBatch, false);
BlockingQueue<BigQueryArrowBatchWrapper> picosBuffer = new LinkedBlockingDeque<>(2);
picosBuffer.add(batchWrapper);
picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true));

Schema bqSchema =
Schema.of(FieldList.of(Field.of("timeStampField", StandardSQLTypeName.TIMESTAMP)));
BigQueryStatement mockStatement = mock(BigQueryStatement.class);
when(mockStatement.isEnableTimestampPicos()).thenReturn(true);

BigQueryArrowResultSet rs =
BigQueryArrowResultSet.of(
bqSchema, arrowSchema, 1, mockStatement, picosBuffer, mock(Future.class), null);

assertThat(rs.next()).isTrue();
// getString returns full 12-digit picosecond string
assertThat(rs.getString("timeStampField")).isEqualTo("2026-04-08 10:00:00.123456789123");
assertThat(rs.getString(1)).isEqualTo("2026-04-08 10:00:00.123456789123");

// getTimestamp returns java.sql.Timestamp with 9 digits of nanoseconds
Timestamp ts = rs.getTimestamp("timeStampField");
assertThat(ts).isNotNull();
assertThat(ts.getNanos()).isEqualTo(123456789);

// getObject returns java.sql.Timestamp
Object obj = rs.getObject(1);
assertThat(obj).isInstanceOf(Timestamp.class);
assertThat(((Timestamp) obj).getNanos()).isEqualTo(123456789);

rs.close();
timeStampPicosVector.close();
picosRoot.close();
allocator.close();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static java.time.Month.MARCH;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FieldList;
Expand Down Expand Up @@ -509,7 +510,8 @@ public void testGetObjectWithType_failure(Object column, Class<?> type) throws S
}

@Test
public void testGetString_timestamp() throws SQLException {
public void testGetString_timestampWithPicoseconds() throws SQLException {
when(statement.isEnableTimestampPicos()).thenReturn(true);
assertThat(resetResultSet()).isTrue();
bigQueryJsonResultSet.next();
assertThat(bigQueryJsonResultSet.getString("fifth")).isEqualTo("2023-03-30 11:14:19.820000");
Expand Down
Loading
Loading