Protobuf + BigQuery + Go

Seamlessly save and load protocol buffers to and from BigQuery using Go.

This library provides add-ons to cloud.google.com/bigquery for first-class protobuf support using protobuf reflection.

Installing

$ go get -u go.einride.tech/protobuf-bigquery

Examples

protobq.InferSchema

BigQuery schema inference for arbitrary protobuf messages.

func ExampleInferSchema() {
    msg := &library.Book{}
    schema := protobq.InferSchema(msg)
    expected := bigquery.Schema{
        {Name: "name", Type: bigquery.StringFieldType},
        {Name: "author", Type: bigquery.StringFieldType},
        {Name: "title", Type: bigquery.StringFieldType},
        {Name: "read", Type: bigquery.BooleanFieldType},
    }
    fmt.Println(cmp.Equal(expected, schema))
    // Output: true
}

protobq.MessageSaver

An implementation of bigquery.ValueSaver that saves arbitrary protobuf messages to BigQuery.

func ExampleMessageSaver() {
    ctx := context.Background()
    // Write protobuf messages to a BigQuery table.
    projectID := flag.String("project", "", "BigQuery project to write to.")
    datasetID := flag.String("dataset", "", "BigQuery dataset to write to.")
    tableID := flag.String("table", "", "BigQuery table to write to.")
    create := flag.Bool("create", false, "Flag indicating whether to create the table.")
    flag.Parse()
    // Connect to BigQuery.
    client, err := bigquery.NewClient(ctx, *projectID)
    if err != nil {
        panic(err) // TODO: Handle error.
    }
    table := client.Dataset(*datasetID).Table(*tableID)
    // Create the table by inferring the BigQuery schema from the protobuf schema.
    if *create {
        if err := table.Create(ctx, &bigquery.TableMetadata{
            Schema: protobq.InferSchema(&publicv1.FilmLocation{}),
        }); err != nil {
            panic(err) // TODO: Handle error.
        }
    }
    // Insert the protobuf messages.
    inserter := table.Inserter()
    for i, filmLocation := range []*publicv1.FilmLocation{
        {Title: "Dark Passage", ReleaseYear: 1947, Locations: "Filbert Steps"},
        {Title: "D.O.A", ReleaseYear: 1950, Locations: "Union Square"},
        {Title: "Flower Drum Song", ReleaseYear: 1961, Locations: "Chinatown"},
    } {
        if err := inserter.Put(ctx, &protobq.MessageSaver{
            Message:  filmLocation,
            InsertID: strconv.Itoa(i), // include an optional insert ID
        }); err != nil {
            panic(err) // TODO: Handle error.
        }
    }
}

protobq.MessageLoader

An implementation of bigquery.ValueLoader that loads arbitrary protobuf messages from BigQuery.

func ExampleMessageLoader() {
    ctx := context.Background()
    // Read from the public "film locations" BigQuery dataset into a proto message.
    const (
        project = "bigquery-public-data"
        dataset = "san_francisco_film_locations"
        table   = "film_locations"
    )
    // Connect to BigQuery.
    client, err := bigquery.NewClient(ctx, project)
    if err != nil {
        panic(err) // TODO: Handle error.
    }
    // Load BigQuery rows into a FilmLocation message.
    messageLoader := &protobq.MessageLoader{
        Message: &publicv1.FilmLocation{},
    }
    // Iterate rows in table.
    rowIterator := client.Dataset(dataset).Table(table).Read(ctx)
    for {
        // Load next row into the FilmLocation message.
        if err := rowIterator.Next(messageLoader); err != nil {
            if errors.Is(err, iterator.Done) {
                break
            }
            panic(err) // TODO: Handle error.
        }
        // Print the message.
        fmt.Println(prototext.Format(messageLoader.Message))
    }
}

Features

Support for Well-Known Types (google.protobuf)

Reference ≫

Support for API Common Protos (google.type)

Reference ≫