Package org.apache.beam.sdk.io.gcp.bigquery

Class TableRowToStorageApiProto

java.lang.Object

org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto

public class TableRowToStorageApiProto
extends java.lang.Object

Utility methods for converting JSON TableRow objects to dynamic protocol message, for use with the Storage write API.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	TableRowToStorageApiProto.ErrorCollector	This class collects errors encountered while attempting to convert a json message into its matching protocol-buffer message.
static class	TableRowToStorageApiProto.SchemaDoesntMatchException
static class	TableRowToStorageApiProto.SchemaInformation
static class	TableRowToStorageApiProto.SchemaMissingRequiredFieldException
static class	TableRowToStorageApiProto.SchemaTooNarrowException
static class	TableRowToStorageApiProto.SingleValueConversionException
static interface	TableRowToStorageApiProto.ThrowingBiConsumer<FirstInputT,SecondInputT>
static interface	TableRowToStorageApiProto.ThrowingBiFunction<FirstInputT,SecondInputT,OutputT>

Constructor Summary

Constructors

Constructor	Description
TableRowToStorageApiProto()

Method Summary

Modifier and Type	Method	Description
static com.google.protobuf.Descriptors.Descriptor	getDescriptorFromTableSchema(com.google.api.services.bigquery.model.TableSchema jsonSchema, boolean respectRequired, boolean includeCdcColumns)
static com.google.protobuf.Descriptors.Descriptor	getDescriptorFromTableSchema(com.google.cloud.bigquery.storage.v1.TableSchema tableSchema, boolean respectRequired, boolean includeCdcColumns)	Given a BigQuery TableSchema, returns a protocol-buffer Descriptor that can be used to write data using the BigQuery Storage API.
static java.lang.Object	jsonValueFromMessageValue(TableRowToStorageApiProto.SchemaInformation schemaInformation, com.google.protobuf.Descriptors.FieldDescriptor fieldDescriptor, java.lang.Object fieldValue, boolean expandRepeated, java.util.function.Predicate<java.lang.String> includeField, java.lang.String prefix, boolean useSetF)
static com.google.protobuf.ByteString	mergeNewFields(com.google.protobuf.ByteString tableRowProto, com.google.protobuf.DescriptorProtos.DescriptorProto descriptorProto, com.google.cloud.bigquery.storage.v1.TableSchema tableSchema, TableRowToStorageApiProto.SchemaInformation schemaInformation, com.google.api.services.bigquery.model.TableRow unknownFields, boolean ignoreUnknownValues)
static @Nullable com.google.protobuf.DynamicMessage	messageFromMap(TableRowToStorageApiProto.SchemaInformation schemaInformation, @Nullable com.google.protobuf.Descriptors.Descriptor descriptor, java.util.AbstractMap<java.lang.String,java.lang.Object> map, boolean ignoreUnknownValues, boolean allowMissingRequiredFields, @Nullable com.google.api.services.bigquery.model.TableRow unknownFields, @Nullable java.lang.String changeType, @Nullable java.lang.String changeSequenceNum, TableRowToStorageApiProto.ErrorCollector collectedExceptions)
static @Nullable com.google.protobuf.DynamicMessage	messageFromTableRow(TableRowToStorageApiProto.SchemaInformation schemaInformation, @Nullable com.google.protobuf.Descriptors.Descriptor descriptor, com.google.api.services.bigquery.model.TableRow tableRow, boolean ignoreUnknownValues, boolean allowMissingRequiredFields, @Nullable com.google.api.services.bigquery.model.TableRow unknownFields, @Nullable java.lang.String changeType, long changeSequenceNum, TableRowToStorageApiProto.ErrorCollector collectedExceptions)	Forwards changeSequenceNum to messageFromTableRow(SchemaInformation, Descriptor, TableRow, boolean, boolean, TableRow, String, String, ErrorCollector) via Long.toHexString(long).
static @Nullable com.google.protobuf.DynamicMessage	messageFromTableRow(TableRowToStorageApiProto.SchemaInformation schemaInformation, @Nullable com.google.protobuf.Descriptors.Descriptor descriptor, com.google.api.services.bigquery.model.TableRow tableRow, boolean ignoreUnknownValues, boolean allowMissingRequiredFields, @Nullable com.google.api.services.bigquery.model.TableRow unknownFields, @Nullable java.lang.String changeType, @Nullable java.lang.String changeSequenceNum, TableRowToStorageApiProto.ErrorCollector collectedExceptions)	Given a BigQuery TableRow, returns a protocol-buffer message that can be used to write data using the BigQuery Storage API.
static com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode	modeToProtoMode(@Nullable java.lang.String defaultValueExpression, java.lang.String mode)
static java.lang.String	protoModeToJsonMode(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode protoMode)
static com.google.api.services.bigquery.model.TableSchema	protoSchemaToTableSchema(com.google.cloud.bigquery.storage.v1.TableSchema protoTableSchema)
static com.google.api.services.bigquery.model.TableFieldSchema	protoTableFieldToTableField(com.google.cloud.bigquery.storage.v1.TableFieldSchema protoTableField)
static java.lang.String	protoTypeToJsonType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type protoType)
static com.google.cloud.bigquery.storage.v1.TableSchema	schemaToProtoTableSchema(com.google.api.services.bigquery.model.TableSchema tableSchema)
static com.google.cloud.bigquery.storage.v1.TableFieldSchema	tableFieldToProtoTableField(com.google.api.services.bigquery.model.TableFieldSchema field)
static com.google.api.services.bigquery.model.TableRow	tableRowFromMessage(TableRowToStorageApiProto.SchemaInformation schemaInformation, com.google.protobuf.Message message, boolean includeCdcColumns, java.util.function.Predicate<java.lang.String> includeField)
static com.google.api.services.bigquery.model.TableRow	tableRowFromMessage(TableRowToStorageApiProto.SchemaInformation schemaInformation, com.google.protobuf.Message message, boolean includeCdcColumns, java.util.function.Predicate<java.lang.String> includeField, java.lang.String namePrefix)
static com.google.api.services.bigquery.model.TableRow	tableRowFromMessageUseSetF(TableRowToStorageApiProto.SchemaInformation schemaInformation, com.google.protobuf.Message message, boolean includeCdcColumns, java.util.function.Predicate<java.lang.String> includeField, java.lang.String namePrefix)
static byte[]	tableSchemaHash(com.google.cloud.bigquery.storage.v1.TableSchema tableSchema)
static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode	tableSchemaHash(java.lang.String prefix, java.util.List<com.google.cloud.bigquery.storage.v1.TableFieldSchema> fields)
static com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type	typeToProtoType(java.lang.String type)
static com.google.protobuf.Descriptors.Descriptor	wrapDescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto descriptorProto)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TableRowToStorageApiProto

public TableRowToStorageApiProto()

Method Detail

tableSchemaHash

public static byte[] tableSchemaHash(com.google.cloud.bigquery.storage.v1.TableSchema tableSchema)

tableSchemaHash

public static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode tableSchemaHash(java.lang.String prefix,
                                                                                                       java.util.List<com.google.cloud.bigquery.storage.v1.TableFieldSchema> fields)

modeToProtoMode

public static com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode modeToProtoMode(@Nullable java.lang.String defaultValueExpression,
                                                                                         java.lang.String mode)

protoModeToJsonMode

public static java.lang.String protoModeToJsonMode(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode protoMode)

protoTypeToJsonType

public static java.lang.String protoTypeToJsonType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type protoType)

typeToProtoType

public static com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type typeToProtoType(java.lang.String type)

protoSchemaToTableSchema

public static com.google.api.services.bigquery.model.TableSchema protoSchemaToTableSchema(com.google.cloud.bigquery.storage.v1.TableSchema protoTableSchema)

protoTableFieldToTableField

public static com.google.api.services.bigquery.model.TableFieldSchema protoTableFieldToTableField(com.google.cloud.bigquery.storage.v1.TableFieldSchema protoTableField)

schemaToProtoTableSchema

public static com.google.cloud.bigquery.storage.v1.TableSchema schemaToProtoTableSchema(com.google.api.services.bigquery.model.TableSchema tableSchema)

tableFieldToProtoTableField

public static com.google.cloud.bigquery.storage.v1.TableFieldSchema tableFieldToProtoTableField(com.google.api.services.bigquery.model.TableFieldSchema field)

getDescriptorFromTableSchema

public static com.google.protobuf.Descriptors.Descriptor getDescriptorFromTableSchema(com.google.api.services.bigquery.model.TableSchema jsonSchema,
                                                                                      boolean respectRequired,
                                                                                      boolean includeCdcColumns)
                                                                               throws com.google.protobuf.Descriptors.DescriptorValidationException

Throws:

com.google.protobuf.Descriptors.DescriptorValidationException

getDescriptorFromTableSchema

public static com.google.protobuf.Descriptors.Descriptor getDescriptorFromTableSchema(com.google.cloud.bigquery.storage.v1.TableSchema tableSchema,
                                                                                      boolean respectRequired,
                                                                                      boolean includeCdcColumns)
                                                                               throws com.google.protobuf.Descriptors.DescriptorValidationException

Given a BigQuery TableSchema, returns a protocol-buffer Descriptor that can be used to write data using the BigQuery Storage API.

Throws:

com.google.protobuf.Descriptors.DescriptorValidationException

wrapDescriptorProto

public static com.google.protobuf.Descriptors.Descriptor wrapDescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto descriptorProto)
                                                                      throws com.google.protobuf.Descriptors.DescriptorValidationException

Throws:

com.google.protobuf.Descriptors.DescriptorValidationException

messageFromMap

public static @Nullable com.google.protobuf.DynamicMessage messageFromMap(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                                          @Nullable com.google.protobuf.Descriptors.Descriptor descriptor,
                                                                          java.util.AbstractMap<java.lang.String,java.lang.Object> map,
                                                                          boolean ignoreUnknownValues,
                                                                          boolean allowMissingRequiredFields,
                                                                          @Nullable com.google.api.services.bigquery.model.TableRow unknownFields,
                                                                          @Nullable java.lang.String changeType,
                                                                          @Nullable java.lang.String changeSequenceNum,
                                                                          TableRowToStorageApiProto.ErrorCollector collectedExceptions)
                                                                   throws org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

Throws:

org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

messageFromTableRow

public static @Nullable com.google.protobuf.DynamicMessage messageFromTableRow(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                                               @Nullable com.google.protobuf.Descriptors.Descriptor descriptor,
                                                                               com.google.api.services.bigquery.model.TableRow tableRow,
                                                                               boolean ignoreUnknownValues,
                                                                               boolean allowMissingRequiredFields,
                                                                               @Nullable com.google.api.services.bigquery.model.TableRow unknownFields,
                                                                               @Nullable java.lang.String changeType,
                                                                               long changeSequenceNum,
                                                                               TableRowToStorageApiProto.ErrorCollector collectedExceptions)
                                                                        throws org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

Forwards changeSequenceNum to messageFromTableRow(SchemaInformation, Descriptor, TableRow, boolean, boolean, TableRow, String, String, ErrorCollector) via Long.toHexString(long).

Throws:

org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

messageFromTableRow

public static @Nullable com.google.protobuf.DynamicMessage messageFromTableRow(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                                               @Nullable com.google.protobuf.Descriptors.Descriptor descriptor,
                                                                               com.google.api.services.bigquery.model.TableRow tableRow,
                                                                               boolean ignoreUnknownValues,
                                                                               boolean allowMissingRequiredFields,
                                                                               @Nullable com.google.api.services.bigquery.model.TableRow unknownFields,
                                                                               @Nullable java.lang.String changeType,
                                                                               @Nullable java.lang.String changeSequenceNum,
                                                                               TableRowToStorageApiProto.ErrorCollector collectedExceptions)
                                                                        throws org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

Given a BigQuery TableRow, returns a protocol-buffer message that can be used to write data using the BigQuery Storage API.

Throws:

org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

mergeNewFields

public static com.google.protobuf.ByteString mergeNewFields(com.google.protobuf.ByteString tableRowProto,
                                                            com.google.protobuf.DescriptorProtos.DescriptorProto descriptorProto,
                                                            com.google.cloud.bigquery.storage.v1.TableSchema tableSchema,
                                                            TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                            com.google.api.services.bigquery.model.TableRow unknownFields,
                                                            boolean ignoreUnknownValues)
                                                     throws org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

Throws:

org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException

tableRowFromMessage

public static com.google.api.services.bigquery.model.TableRow tableRowFromMessage(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                                                  com.google.protobuf.Message message,
                                                                                  boolean includeCdcColumns,
                                                                                  java.util.function.Predicate<java.lang.String> includeField)

tableRowFromMessage

public static com.google.api.services.bigquery.model.TableRow tableRowFromMessage(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                                                  com.google.protobuf.Message message,
                                                                                  boolean includeCdcColumns,
                                                                                  java.util.function.Predicate<java.lang.String> includeField,
                                                                                  java.lang.String namePrefix)

tableRowFromMessageUseSetF

public static com.google.api.services.bigquery.model.TableRow tableRowFromMessageUseSetF(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                                                         com.google.protobuf.Message message,
                                                                                         boolean includeCdcColumns,
                                                                                         java.util.function.Predicate<java.lang.String> includeField,
                                                                                         java.lang.String namePrefix)

jsonValueFromMessageValue

public static java.lang.Object jsonValueFromMessageValue(TableRowToStorageApiProto.SchemaInformation schemaInformation,
                                                         com.google.protobuf.Descriptors.FieldDescriptor fieldDescriptor,
                                                         java.lang.Object fieldValue,
                                                         boolean expandRepeated,
                                                         java.util.function.Predicate<java.lang.String> includeField,
                                                         java.lang.String prefix,
                                                         boolean useSetF)