Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/beam_StressTests_Java_BigTableIO.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ jobs:
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
- name: run BigTable StressTest Large
- name: run Bigtable StressTest Large
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :it:google-cloud-platform:BigTableStressTestLarge --info -DinfluxHost="http://10.128.0.96:8086" -DinfluxDatabase="beam_test_metrics" -DinfluxMeasurement="java_stress_test_bigtable"
4 changes: 2 additions & 2 deletions .test-infra/tools/refresh_looker_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
LOOKS_TO_DOWNLOAD = [
("30", ["18", "50", "92", "49", "91"]), # BigQueryIO_Read
("31", ["19", "52", "88", "51", "87"]), # BigQueryIO_Write
("32", ["20", "60", "104", "59", "103"]), # BigTableIO_Read
("33", ["21", "70", "116", "69", "115"]), # BigTableIO_Write
("32", ["20", "60", "104", "59", "103"]), # BigtableIO_Read
("33", ["21", "70", "116", "69", "115"]), # BigtableIO_Write
("34", ["22", "56", "96", "55", "95"]), # TextIO_Read
("35", ["23", "64", "110", "63", "109"]), # TextIO_Write
("113", ["386", "388", "390", "392", "394"]), # IcebergIO_Read
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
* <ul>
* <li>File system (Only JSON or CSV)
* <li><a href=https://cloud.google.com/bigquery>Google Cloud BigQuery</a>
* <li><a href=https://cloud.google.com/bigtable>Cloud BigTable</a>
* <li><a href=https://cloud.google.com/bigtable>Cloud Bigtable</a>
* </ul>
* <li>A configured tokenization server
* </ul>
Expand Down Expand Up @@ -138,13 +138,13 @@
* - <b><i>bigQueryTableName</i></b>: Cloud BigQuery table name to write into
* - <b><i>tempLocation</i></b>: Folder in a Google Cloud Storage bucket, which is needed for
* BigQuery to handle data writing
* - Cloud BigTable
* - <b><i>bigTableProjectId</i></b>: Id of the project where the Cloud BigTable instance to write into
* - Cloud Bigtable
* - <b><i>bigTableProjectId</i></b>: Id of the project where the Cloud Bigtable instance to write into
* is located
* - <b><i>bigTableInstanceId</i></b>: Id of the Cloud BigTable instance to write into
* - <b><i>bigTableTableId</i></b>: Id of the Cloud BigTable table to write into
* - <b><i>bigTableKeyColumnName</i></b>: Column name to use as a key in Cloud BigTable
* - <b><i>bigTableColumnFamilyName</i></b>: Column family name to use in Cloud BigTable
* - <b><i>bigTableInstanceId</i></b>: Id of the Cloud Bigtable instance to write into
* - <b><i>bigTableTableId</i></b>: Id of the Cloud Bigtable table to write into
* - <b><i>bigTableKeyColumnName</i></b>: Column name to use as a key in Cloud Bigtable
* - <b><i>bigTableColumnFamilyName</i></b>: Column family name to use in Cloud Bigtable
* - RPC server parameters
* - <b><i>rpcUri</i></b>: URI for the API calls to RPC server
* - <b><i>batchSize</i></b>: Size of the batch to send to RPC server per request
Expand Down Expand Up @@ -335,7 +335,7 @@ public static PipelineResult run(DataTokenizationOptions options) {
.write(tokenizedRows.get(TOKENIZATION_OUT), schema.getBeamSchema());
} else {
throw new IllegalStateException(
"No sink is provided, please configure BigQuery or BigTable.");
"No sink is provided, please configure BigQuery or Bigtable.");
}

return pipeline.run();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Supported destination sinks:

- File system
- [Google Cloud BigQuery](https://cloud.google.com/bigquery)
- [Cloud BigTable](https://cloud.google.com/bigtable)
- [Cloud Bigtable](https://cloud.google.com/bigtable)

Supported data schema format:

Expand Down Expand Up @@ -122,13 +122,13 @@ To execute this pipeline, specify the parameters:
- **bigQueryTableName**: Cloud BigQuery table name to write into
- **tempLocation**: Folder in a Google Cloud Storage bucket, which is needed for
BigQuery to handle data writing
- Cloud BigTable
- **bigTableProjectId**: Id of the project where the Cloud BigTable instance to write into
- Cloud Bigtable
- **bigTableProjectId**: Id of the project where the Cloud Bigtable instance to write into
is located
- **bigTableInstanceId**: Id of the Cloud BigTable instance to write into
- **bigTableTableId**: Id of the Cloud BigTable table to write into
- **bigTableKeyColumnName**: Column name to use as a key in Cloud BigTable
- **bigTableColumnFamilyName**: Column family name to use in Cloud BigTable
- **bigTableInstanceId**: Id of the Cloud Bigtable instance to write into
- **bigTableTableId**: Id of the Cloud Bigtable table to write into
- **bigTableKeyColumnName**: Column name to use as a key in Cloud Bigtable
- **bigTableColumnFamilyName**: Column family name to use in Cloud Bigtable
- RPC server parameters
- **rpcUri**: URI for the API calls to RPC server
- **batchSize**: Size of the batch to send to RPC server per request
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** The {@link TokenizationBigQueryIO} class for writing data from template to BigTable. */
/** The {@link TokenizationBigQueryIO} class for writing data from template to BigQuery. */
public class TokenizationBigQueryIO {

/** Logger for class. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** The {@link TokenizationBigTableIO} class for writing data from template to BigTable. */
/** The {@link TokenizationBigTableIO} class for writing data from template to Bigtable. */
public class TokenizationBigTableIO {

/** Logger for class. */
Expand Down Expand Up @@ -101,7 +101,7 @@ public void processElement(
.build())
.build())
.collect(Collectors.toSet());
// Converting key value to BigTable format
// Converting key value to Bigtable format
String columnName = in.getString(options.getBigTableKeyColumnName());
if (columnName != null) {
ByteString key = ByteString.copyFrom(columnName, StandardCharsets.UTF_8);
Expand All @@ -128,31 +128,31 @@ public void processElement(@Element BigtableWriteResult in) {

/**
* Necessary {@link PipelineOptions} options for Pipelines that perform write operations to
* BigTable.
* Bigtable.
*/
public interface BigTableOptions extends PipelineOptions {

@Description("Id of the project where the Cloud BigTable instance to write into is located.")
@Description("Id of the project where the Cloud Bigtable instance to write into is located.")
String getBigTableProjectId();

void setBigTableProjectId(String bigTableProjectId);

@Description("Id of the Cloud BigTable instance to write into.")
@Description("Id of the Cloud Bigtable instance to write into.")
String getBigTableInstanceId();

void setBigTableInstanceId(String bigTableInstanceId);

@Description("Id of the Cloud BigTable table to write into.")
@Description("Id of the Cloud Bigtable table to write into.")
String getBigTableTableId();

void setBigTableTableId(String bigTableTableId);

@Description("Column name to use as a key in Cloud BigTable.")
@Description("Column name to use as a key in Cloud Bigtable.")
String getBigTableKeyColumnName();

void setBigTableKeyColumnName(String bigTableKeyColumnName);

@Description("Column family name to use in Cloud BigTable.")
@Description("Column family name to use in Cloud Bigtable.")
String getBigTableColumnFamilyName();

void setBigTableColumnFamilyName(String bigTableColumnFamilyName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
import org.junit.Test;

/**
* BigTableIO performance tests.
* BigtableIO performance tests.
*
* <p>Example trigger command for all tests: "mvn test -pl it/google-cloud-platform -am
* -Dtest=BigTableIOLT \ -Dproject=[gcpProject] -DartifactBucket=[temp bucket]
Expand Down Expand Up @@ -248,7 +248,7 @@ abstract static class Builder {
abstract Builder toBuilder();
}

/** Maps long number to the BigTable format record. */
/** Maps long number to the Bigtable format record. */
private static class MapToBigTableFormat extends DoFn<Long, KV<ByteString, Iterable<Mutation>>>
implements Serializable {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
import org.junit.Test;

/**
* BigTableIO stress test. The test is designed to assess the performance of BigTableIO under
* BigtableIO stress test. The test is designed to assess the performance of BigtableIO under
* various conditions.
*
* <p>Usage: <br>
Expand Down Expand Up @@ -229,7 +229,7 @@ public void runTest() throws IOException, ParseException, InterruptedException {
}

/**
* The method creates a pipeline to simulate data generation and write operations to BigTable,
* The method creates a pipeline to simulate data generation and write operations to Bigtable,
* based on the specified configuration parameters. The stress test involves varying the load
* dynamically over time, with options to use configurable parameters.
*/
Expand Down Expand Up @@ -278,7 +278,7 @@ private PipelineLauncher.LaunchInfo generateDataAndWrite() throws IOException {
return pipelineLauncher.launch(project, region, options);
}

/** The method reads data from BigTable in batch mode. */
/** The method reads data from Bigtable in batch mode. */
private PipelineLauncher.LaunchInfo readData() throws IOException {
BigtableIO.Read readIO =
BigtableIO.read()
Expand Down Expand Up @@ -307,7 +307,7 @@ private PipelineLauncher.LaunchInfo readData() throws IOException {
return pipelineLauncher.launch(project, region, options);
}

/** Options for BigTableIO stress test. */
/** Options for BigtableIO stress test. */
static class Configuration extends SyntheticSourceOptions {
/** Pipeline timeout in minutes. Must be a positive value. */
@JsonProperty public int pipelineTimeout = 20;
Expand Down Expand Up @@ -347,7 +347,7 @@ static class Configuration extends SyntheticSourceOptions {
@JsonProperty public String influxDatabase;
}

/** Maps Instant to the BigTable format record. */
/** Maps Instant to the Bigtable format record. */
private static class MapToBigTableFormat
extends DoFn<KV<byte[], byte[]>, KV<ByteString, Iterable<Mutation>>> implements Serializable {

Expand Down
2 changes: 1 addition & 1 deletion learning/prompts/code-generation/04_io_bigtable.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ if __name__ == "__main__":
```
The `ReadFromBigtable` transform returns a `PCollection` of `PartialRowData` objects, each representing a Bigtable row. For more information about this row object, see [PartialRowData (row_key)](https://cloud.google.com/python/docs/reference/bigtable/latest/row#class-googlecloudbigtablerowpartialrowdatarowkey).

For more information, see the [BigTable I/O connector documentation](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigtableio.html).
For more information, see the [Bigtable I/O connector documentation](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigtableio.html).

For samples that show common pipeline configurations, see [Pipeline option patterns](https://beam.apache.org/documentation/patterns/pipeline-options/).

4 changes: 2 additions & 2 deletions learning/prompts/code-generation/java/04_io_bigtable.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ Prompt:
Write a sample Java code snippet that writes data to a Google Bigtable table using Apache Beam.

Response:
Your Apache Beam pipeline can write data to a Bigtable table using the Apache Beam BigTableIO connector.
Your Apache Beam pipeline can write data to a Bigtable table using the Apache Beam BigtableIO connector.

Here is an example of how to use the BigTableIO connector to accomplish this:
Here is an example of how to use the BigtableIO connector to accomplish this:

```java
package bigtable;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
* after A, up to but not including the first record starting at or after B".
*
* <p>Some examples of such sources include reading lines or CSV from a text file, reading keys and
* values from a BigTable, etc.
* values from a Bigtable, etc.
*
* <p>The concept of <i>split points</i> allows to extend the definitions for dealing with sources
* where some records cannot be identified by a unique starting position.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
/**
* {@link TableProvider} for {@link BigtableTable}.
*
* <p>A sample of BigTable table is:
* <p>A sample of Bigtable table is:
*
* <pre>{@code
* CREATE EXTERNAL TABLE beamTable(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@
* limitations under the License.
*/

/** Table schema for BigTable. */
/** Table schema for Bigtable. */
package org.apache.beam.sdk.extensions.sql.meta.provider.bigtable;
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,7 @@ long flush(
}
if (schemaMismatchError) {
LOG.info(
"Vortex failed stream open due to incompatible fields. This is likely because the BigTable "
"Vortex failed stream open due to incompatible fields. This is likely because the Bigtable "
+ "schema was recently updated and Vortex hasn't noticed yet, so retrying. error {}",
Preconditions.checkStateNotNull(error).toString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,12 @@ abstract Builder setBigtableOptionsConfigurator(
}

public BigtableConfig withProjectId(ValueProvider<String> projectId) {
checkArgument(projectId != null, "Project Id of BigTable can not be null");
checkArgument(projectId != null, "Project Id of Bigtable can not be null");
return toBuilder().setProjectId(projectId).build();
}

public BigtableConfig withInstanceId(ValueProvider<String> instanceId) {
checkArgument(instanceId != null, "Instance Id of BigTable can not be null");
checkArgument(instanceId != null, "Instance Id of Bigtable can not be null");
return toBuilder().setInstanceId(instanceId).build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
/**
* {@link PTransform Transforms} for reading from and writing to Google Cloud Bigtable.
*
* <p>Please note the Cloud BigTable HBase connector available <a
* <p>Please note the Cloud Bigtable HBase connector available <a
* href="https://github.com/googleapis/java-bigtable-hbase/tree/master/bigtable-dataflow-parent/bigtable-hbase-beam">here</a>.
* We recommend using that connector over this one if <a
* href="https://cloud.google.com/bigtable/docs/hbase-bigtable">HBase API</a></> works for your
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1641,7 +1641,7 @@ public void testGetSplitPointsConsumed() throws Exception {
}

@Test
public void testReadWithBigTableOptionsSetsRetryOptions() {
public void testReadWithBigtableOptionsSetsRetryOptions() {
final int initialBackoffMillis = -1;

BigtableOptions.Builder optionsBuilder = BIGTABLE_OPTIONS.toBuilder();
Expand All @@ -1660,7 +1660,7 @@ public void testReadWithBigTableOptionsSetsRetryOptions() {
}

@Test
public void testWriteWithBigTableOptionsSetsBulkOptionsAndRetryOptions() {
public void testWriteWithBigtableOptionsSetsBulkOptionsAndRetryOptions() {
final int maxInflightRpcs = 1;
final int initialBackoffMillis = -1;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def enrichment_with_bigtable():
_ = (
p
| "Create" >> beam.Create(data)
| "Enrich W/ BigTable" >> Enrichment(bigtable_handler)
| "Enrich W/ Bigtable" >> Enrichment(bigtable_handler)
| "Print" >> beam.Map(print))
# [END enrichment_with_bigtable]

Expand Down
10 changes: 5 additions & 5 deletions sdks/python/apache_beam/io/gcp/bigtableio.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,18 @@
# limitations under the License.
#

"""BigTable connector
"""Bigtable connector

This module implements writing to BigTable tables.
The default mode is to set row data to write to BigTable tables.
This module implements writing to Bigtable tables.
The default mode is to set row data to write to Bigtable tables.
The syntax supported is described here:
https://cloud.google.com/bigtable/docs/quickstart-cbt

BigTable connector can be used as main outputs. A main output
Bigtable connector can be used as main outputs. A main output
(common case) is expected to be massive and will be split into
manageable chunks and processed in parallel. In the example below
we created a list of rows then passed to the GeneratedDirectRows
DoFn to set the Cells and then we call the BigTableWriteFn to insert
DoFn to set the Cells and then we call the _BigTableWriteFn to insert
those generated rows in the table.

main_table = (p
Expand Down
4 changes: 2 additions & 2 deletions sdks/python/apache_beam/io/gcp/bigtableio_it_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

"""Integration tests for BigTable service."""
"""Integration tests for Bigtable service."""

import logging
import os
Expand Down Expand Up @@ -66,7 +66,7 @@ def instance_prefix(instance):
os.environ.get('TRANSFORM_SERVICE_PORT'),
"A valid expansion service is not available for executing the "
"cross-language test.")
class TestReadFromBigTableIT(unittest.TestCase):
class TestReadFromBigtableIT(unittest.TestCase):
INSTANCE = "bt-read-tests"
TABLE_ID = "test-table"

Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/io/gcp/bigtableio_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

"""Unit tests for BigTable service."""
"""Unit tests for Bigtable service."""

import logging
import string
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/io/iobase.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ class RangeTracker(object):
at or after 'B'".

Some examples of such sources include reading lines or CSV from a text file,
reading keys and values from a BigTable, etc.
reading keys and values from a Bigtable, etc.

The concept of *split points* allows to extend the definitions for dealing
with sources where some records cannot be identified by a unique starting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ bigquery_io_json_file_loads_write:

bigtable_io_read:
test_description: |
BigTableIO read test 100 GB.
BigtableIO read test 100 GB.
project: apache-beam-testing
metrics_dataset: performance_tests
metrics_table: io_performance_metrics
Expand All @@ -197,7 +197,7 @@ bigtable_io_read:

bigtable_io_write:
test_description: |
BigTableIO write test 100 GB.
BigtableIO write test 100 GB.
project: apache-beam-testing
metrics_dataset: performance_tests
metrics_table: io_performance_metrics
Expand Down
Loading
Loading