Lokasi ngalangkungan proxy:   [ UP ]  
[Ngawartoskeun bug]   [Panyetelan cookie]                
Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit 5701597

Browse files
dark0davedark0dave
authored
fix: even better ISO compilance (#1607)
Using docs from google https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#data_types as guide, for how types should be parsed Signed-off-by: dark0dave <dark0dave@mykolab.com>
1 parent 3148c3a commit 5701597

3 files changed

Lines changed: 60 additions & 10 deletions

File tree

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.threeten.bp.ZoneOffset;
4040
import org.threeten.bp.format.DateTimeFormatter;
4141
import org.threeten.bp.format.DateTimeFormatterBuilder;
42+
import org.threeten.bp.format.TextStyle;
4243
import org.threeten.bp.temporal.ChronoField;
4344
import org.threeten.bp.temporal.TemporalAccessor;
4445

@@ -61,7 +62,7 @@ public class JsonToProtoMessage {
6162
private static final DateTimeFormatter timestampFormatter =
6263
new DateTimeFormatterBuilder()
6364
.parseLenient()
64-
.append(DateTimeFormatter.ISO_LOCAL_DATE)
65+
.append(DateTimeFormatter.ofPattern("yyyy[/][-]MM[/][-]dd"))
6566
.optionalStart()
6667
.appendLiteral('T')
6768
.optionalEnd()
@@ -74,10 +75,24 @@ public class JsonToProtoMessage {
7475
.optionalStart()
7576
.appendLiteral(':')
7677
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
78+
.optionalEnd()
79+
.optionalStart()
80+
.appendValue(ChronoField.MILLI_OF_SECOND, 3)
81+
.optionalEnd()
82+
.optionalStart()
83+
.appendFraction(ChronoField.MICRO_OF_SECOND, 3, 6, true)
84+
.optionalEnd()
7785
.optionalStart()
7886
.appendFraction(ChronoField.NANO_OF_SECOND, 6, 9, true)
87+
.optionalEnd()
7988
.optionalStart()
80-
.appendOffset("+HHMM", "+00:00")
89+
.appendLiteral(' ')
90+
.optionalEnd()
91+
.optionalStart()
92+
.appendOffset("+HH:MM", "+00:00")
93+
.optionalEnd()
94+
.optionalStart()
95+
.appendZoneText(TextStyle.SHORT)
8196
.optionalEnd()
8297
.optionalStart()
8398
.appendLiteral('Z')
@@ -336,6 +351,11 @@ private static void fillField(
336351
}
337352
} else if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
338353
if (val instanceof String) {
354+
Double parsed = Doubles.tryParse((String) val);
355+
if (parsed != null) {
356+
protoMsg.setField(fieldDescriptor, parsed.longValue() * 10000000);
357+
return;
358+
}
339359
TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
340360
protoMsg.setField(
341361
fieldDescriptor,
@@ -345,6 +365,9 @@ private static void fillField(
345365
} else if (val instanceof Long) {
346366
protoMsg.setField(fieldDescriptor, (Long) val);
347367
return;
368+
} else if (val instanceof Integer) {
369+
protoMsg.setField(fieldDescriptor, new Long((Integer) val) * 10000000);
370+
return;
348371
}
349372
}
350373
}
@@ -549,13 +572,20 @@ private static void fillRepeatedField(
549572
} else if (fieldSchema != null
550573
&& fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
551574
if (val instanceof String) {
552-
TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
553-
protoMsg.addRepeatedField(
554-
fieldDescriptor,
555-
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
556-
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
575+
Double parsed = Doubles.tryParse((String) val);
576+
if (parsed != null) {
577+
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue() * 10000000);
578+
} else {
579+
TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
580+
protoMsg.addRepeatedField(
581+
fieldDescriptor,
582+
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
583+
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
584+
}
557585
} else if (val instanceof Long) {
558586
protoMsg.addRepeatedField(fieldDescriptor, (Long) val);
587+
} else if (val instanceof Integer) {
588+
protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val) * 10000000);
559589
} else {
560590
fail = true;
561591
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -664,17 +664,32 @@ public void testTimestamp() throws Exception {
664664
.addFields(
665665
TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_string_T_Z").build())
666666
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_long").build())
667+
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_int").build())
668+
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_float").build())
669+
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_offset").build())
670+
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_timezone").build())
671+
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_saformat").build())
667672
.build();
668673
TestTimestamp expectedProto =
669674
TestTimestamp.newBuilder()
670675
.setTestString(10L)
671-
.setTestStringTZ(1648493279000000L)
676+
.setTestStringTZ(1648493279010000L)
672677
.setTestLong(0L)
678+
.setTestInt(1534806950000000L)
679+
.setTestFloat(1534680695000000000L)
680+
.setTestOffset(1649135171000000L)
681+
.setTestTimezone(1649174771000000L)
682+
.setTestSaformat(1534680660000000L)
673683
.build();
674684
JSONObject json = new JSONObject();
675685
json.put("test_string", "1970-01-01 00:00:00.000010");
676-
json.put("test_string_T_Z", "2022-03-28T18:47:59.00Z");
686+
json.put("test_string_T_Z", "2022-03-28T18:47:59.01Z");
677687
json.put("test_long", 0L);
688+
json.put("test_int", 153480695);
689+
json.put("test_float", "1.534680695e11");
690+
json.put("test_offset", "2022-04-05T09:06:11+04:00");
691+
json.put("test_timezone", "2022-04-05 09:06:11 PST");
692+
json.put("test_saformat", "2018/08/19 12:11");
678693
DynamicMessage protoMsg =
679694
JsonToProtoMessage.convertJsonToProtoMessage(
680695
TestTimestamp.getDescriptor(), tableSchema, json);
@@ -860,7 +875,7 @@ public void testStructComplex() throws Exception {
860875
.setTestNumeric(
861876
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456")))
862877
.setTestGeo("POINT(1,1)")
863-
.setTestTimestamp(12345678)
878+
.setTestTimestamp(123456780000000L)
864879
.setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)))
865880
.setTestTimeStr(89332507144L)
866881
.addTestNumericRepeated(

‎google-cloud-bigquerystorage/src/test/proto/jsonTest.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,11 @@ message TestTimestamp {
138138
optional int64 test_string = 1;
139139
optional int64 test_string_t_z = 2;
140140
optional int64 test_long = 3;
141+
optional int64 test_int = 4;
142+
optional int64 test_float = 5;
143+
optional int64 test_offset = 6;
144+
optional int64 test_timezone = 7;
145+
optional int64 test_saformat = 8;
141146
}
142147

143148
message TestDate {

0 commit comments

Comments
 (0)