Skip to content

Commit c8e39e1

Browse files
ostinruostinru
authored
Automation: run HBASE in standalone mode (#95)
Start HBase only for test groups that require it (hbase, proxy) instead of launching it as part of the default Hadoop stack. This frees RAM for all other test groups. Additionally, run HBase in standalone mode: * No separate RegionServer process * No separate ZooKeeper process * No dependency on HDFS
1 parent ccea789 commit c8e39e1

23 files changed

Lines changed: 97 additions & 405 deletions

File tree

‎automation/src/main/java/org/apache/cloudberry/pxf/automation/components/hbase/HBase.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.apache.hadoop.hbase.HConstants;
1717
import org.apache.hadoop.hbase.HTableDescriptor;
1818
import org.apache.hadoop.hbase.TableName;
19+
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
1920
import org.apache.hadoop.hbase.client.Admin;
2021
import org.apache.hadoop.hbase.client.ClusterConnection;
2122
import org.apache.hadoop.hbase.client.Connection;
@@ -74,10 +75,10 @@ public void init() throws Exception {
7475
if (StringUtils.isNotEmpty(hbaseRoot)) {
7576
config.addResource(new Path(getHbaseRoot() + "/conf/hbase-site.xml"));
7677
} else {
77-
config.set("hbase.rootdir", "hdfs://" + host + ":8020/hbase");
78+
throw new RuntimeException("No path to hbase-site specified. Please configure hbaseRoot property.");
7879
}
7980

80-
HBaseAdmin.checkHBaseAvailable(config);
81+
waitHBaseAvailable(config);
8182
connection = ConnectionFactory.createConnection(config);
8283
admin = connection.getAdmin();
8384
if (admin.getClusterStatus().getServersSize() == 0) {
@@ -90,6 +91,21 @@ public void init() throws Exception {
9091
ReportUtils.stopLevel(report);
9192
}
9293

94+
private void waitHBaseAvailable(Configuration config) throws Exception {
95+
int attemptsLeft = 50;
96+
while (attemptsLeft > 0) {
97+
try {
98+
HBaseAdmin.checkHBaseAvailable(config);
99+
return;
100+
} catch (IOException e) {
101+
Thread.sleep(500);
102+
attemptsLeft--;
103+
ReportUtils.report(report, getClass(), "HBase availability probe failed: " + e.getMessage(), Reporter.WARNING);
104+
}
105+
}
106+
throw new RuntimeException("HBase did not become available after 50 attempts (25s)");
107+
}
108+
93109
@Override
94110
public void close() {
95111
if (admin != null) {

‎automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hbase/HBaseTest.java‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ private void prepareLookupTable() throws Exception {
152152
*
153153
* @throws Exception if test fails to run
154154
*/
155-
@Test(groups = { "hbase", "features", "sanity", "gpdb" })
155+
@Test(groups = { "hbase" })
156156
public void sanity() throws Exception {
157157

158158
verifyFilterResults(hbaseTable, exTable, "", NO_FILTER, "sanity", false);
@@ -164,7 +164,7 @@ public void sanity() throws Exception {
164164
* @throws Exception if test fails to run
165165
*/
166166
// TODO: pxf_regress shows diff for this test. Should be fixed.
167-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
167+
@Test(enabled = false, groups = { "hbase" })
168168
public void lowerFilter() throws Exception {
169169

170170
String whereClause = " WHERE \"cf1:q3\" < '00000030'";
@@ -178,7 +178,7 @@ public void lowerFilter() throws Exception {
178178
* @throws Exception if test fails to run
179179
*/
180180
// TODO: pxf_regress shows diff for this test. Should be fixed.
181-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
181+
@Test(enabled = false, groups = { "hbase" })
182182
public void rangeFilter() throws Exception {
183183

184184
String whereClause = " WHERE \"cf1:q3\" > '00000090' AND \"cf1:q3\" <= '00000103'";
@@ -192,7 +192,7 @@ public void rangeFilter() throws Exception {
192192
* @throws Exception if test fails to run
193193
*/
194194
// TODO: pxf_regress shows diff for this test. Should be fixed.
195-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
195+
@Test(enabled = false, groups = { "hbase" })
196196
public void specificRowFilter() throws Exception {
197197

198198
String whereClause = " WHERE \"cf1:q3\" = 4";
@@ -206,7 +206,7 @@ public void specificRowFilter() throws Exception {
206206
* @throws Exception if test fails to run
207207
*/
208208
// TODO: pxf_regress shows diff for this test. Should be fixed.
209-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
209+
@Test(enabled = false, groups = { "hbase" })
210210
public void notEqualsFilter() throws Exception {
211211

212212
String whereClause = " WHERE \"cf1:q3\" != 30";
@@ -220,7 +220,7 @@ public void notEqualsFilter() throws Exception {
220220
* @throws Exception if test fails to run
221221
*/
222222
// TODO: pxf_regress shows diff for this test. Should be fixed.
223-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
223+
@Test(enabled = false, groups = { "hbase" })
224224
public void rowkeyEqualsFilter() throws Exception {
225225

226226
String whereClause = " WHERE recordkey = '00000090'";
@@ -234,7 +234,7 @@ public void rowkeyEqualsFilter() throws Exception {
234234
* @throws Exception if test fails to run
235235
*/
236236
// TODO: pxf_regress shows diff for this test. Should be fixed.
237-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
237+
@Test(enabled = false, groups = { "hbase" })
238238
public void rowkeyRangeFilter() throws Exception {
239239

240240
String whereClause = " WHERE recordkey > '00000090' AND recordkey <= '00000103'";
@@ -248,7 +248,7 @@ public void rowkeyRangeFilter() throws Exception {
248248
* @throws Exception if test fails to run
249249
*/
250250
// TODO: pxf_regress shows diff for this test. Should be fixed.
251-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
251+
@Test(enabled = false, groups = { "hbase" })
252252
public void multipleQualifiersPushdownFilter() throws Exception {
253253

254254
String whereClause = " WHERE recordkey != '00000002' AND \"cf1:q3\" > 6 AND \"cf1:q8\" < 10 AND \"cf1:q9\" > 0";
@@ -263,7 +263,7 @@ public void multipleQualifiersPushdownFilter() throws Exception {
263263
*
264264
* @throws Exception if test fails to run
265265
*/
266-
@Test(groups = { "hbase", "features", "gpdb" })
266+
@Test(groups = { "hbase" })
267267
public void partialFilterPushdown() throws Exception {
268268

269269
String whereClause = " WHERE \"cf1:q3\" > 6 AND \"cf1:q7\" = '42'";
@@ -279,7 +279,7 @@ public void partialFilterPushdown() throws Exception {
279279
* @throws Exception if test fails to run
280280
*/
281281
// TODO: pxf_regress shows diff for this test. Should be fixed.
282-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
282+
@Test(enabled = false, groups = { "hbase" })
283283
public void textFilter() throws Exception {
284284

285285
String whereClause = " WHERE \"cf1:q2\" = 'UTF8_計算機用語_00000024'";
@@ -292,7 +292,7 @@ public void textFilter() throws Exception {
292292
*
293293
* @throws Exception if test fails to run
294294
*/
295-
@Test(groups = { "hbase", "features", "gpdb" })
295+
@Test(groups = { "hbase" })
296296
public void doubleFilter() throws Exception {
297297

298298
String whereClause = " WHERE \"cf1:q5\" > 91.92 AND \"cf1:q6\" <= 99999999.99";
@@ -306,7 +306,7 @@ public void doubleFilter() throws Exception {
306306
*
307307
* @throws Exception if test fails to run
308308
*/
309-
@Test(groups = { "hbase", "features", "gpdb" })
309+
@Test(groups = { "hbase" })
310310
public void orFilter() throws Exception {
311311

312312
String whereClause = " WHERE \"cf1:q3\" < 10 OR \"cf1:q5\" > 90";
@@ -323,7 +323,7 @@ public void orFilter() throws Exception {
323323
*
324324
* @throws Exception if test fails to run
325325
*/
326-
@Test(groups = { "hbase", "features", "gpdb" })
326+
@Test(groups = { "hbase" })
327327
public void mixedFilterPushdownOrAnd() throws Exception {
328328

329329
String whereClause = " WHERE (\"cf1:q3\" < 10 OR \"cf1:q5\" > 90) AND (\"cf1:q3\" > 5 AND \"cf1:q8\" < 30)";
@@ -344,7 +344,7 @@ public void mixedFilterPushdownOrAnd() throws Exception {
344344
*
345345
* @throws Exception if test fails to run
346346
*/
347-
@Test(groups = { "hbase", "features", "gpdb" })
347+
@Test(groups = { "hbase" })
348348
public void isNullFilter() throws Exception {
349349

350350
String whereClause = " WHERE \"cf1:q3\" is null";
@@ -358,7 +358,7 @@ public void isNullFilter() throws Exception {
358358
*
359359
* @throws Exception if test fails to run
360360
*/
361-
@Test(groups = { "hbase", "features", "gpdb" })
361+
@Test(groups = { "hbase" })
362362
public void differentColumnNames() throws Exception {
363363

364364
exTableNullHBase = TableFactory.getPxfHBaseReadableTable(
@@ -377,7 +377,7 @@ public void differentColumnNames() throws Exception {
377377
*
378378
* @throws Exception if test fails to run
379379
*/
380-
@Test(groups = { "hbase", "features", "gpdb" })
380+
@Test(groups = { "hbase" })
381381
public void disableLookupTable() throws Exception {
382382

383383
try {
@@ -396,7 +396,7 @@ public void disableLookupTable() throws Exception {
396396
*
397397
* @throws Exception if test fails to run
398398
*/
399-
@Test(groups = { "hbase", "features", "gpdb" })
399+
@Test(groups = { "hbase" })
400400
public void noLookupTable() throws Exception {
401401

402402
try {
@@ -415,7 +415,7 @@ public void noLookupTable() throws Exception {
415415
*
416416
* @throws Exception if test fails to run
417417
*/
418-
@Test(groups = { "hbase", "features", "gpdb" })
418+
@Test(groups = { "hbase" })
419419
public void removeColumnFromLookupTable() throws Exception {
420420

421421
try {
@@ -435,7 +435,7 @@ public void removeColumnFromLookupTable() throws Exception {
435435
* @throws Exception if test fails to run
436436
*/
437437
// TODO: pxf_regress shows diff for this test. Should be fixed.
438-
@Test(enabled = false, groups = { "hbase", "features", "gpdb" })
438+
@Test(enabled = false, groups = { "hbase" })
439439
public void recordkeyAsInteger() throws Exception {
440440

441441
// create external table with record key as INTEGER
@@ -457,7 +457,7 @@ public void recordkeyAsInteger() throws Exception {
457457
*
458458
* @throws Exception if test fails to run
459459
*/
460-
@Test(groups = { "hbase", "features", "gpdb" })
460+
@Test(groups = { "hbase" })
461461
public void notExistingHBaseTable() throws Exception {
462462

463463
ReadableExternalTable notExistsHBaseTableExtTable = TableFactory.getPxfHBaseReadableTable(
@@ -474,7 +474,7 @@ public void notExistingHBaseTable() throws Exception {
474474
*
475475
* @throws Exception if test fails to run
476476
*/
477-
@Test(groups = { "hbase", "features", "gpdb" })
477+
@Test(groups = { "hbase" })
478478
public void multiRegionsData() throws Exception {
479479

480480
HBaseTable multiDataHBaseTable = new HBaseTable(
@@ -513,7 +513,7 @@ public void defaultAnalyze() throws Exception {
513513
*
514514
* @throws Exception if test fails to run
515515
*/
516-
@Test(groups = { "hbase", "features", "gpdb" })
516+
@Test(groups = { "hbase" })
517517
public void longHBaseQualifierNameNoLookupTable() throws Exception {
518518

519519
HBaseTable longQualifiersNamesHBaseTable = new HBaseTable(
@@ -559,7 +559,7 @@ public void longHBaseQualifierNameNoLookupTable() throws Exception {
559559
*
560560
* @throws Exception if test fails to run
561561
*/
562-
@Test(groups = { "hbase", "features", "gpdb" })
562+
@Test(groups = { "hbase" })
563563
public void longHBaseQualifierNameUsingLookupTable() throws Exception {
564564

565565
HBaseTable longQualifiersNamesHBaseTable = new HBaseTable(
@@ -600,7 +600,7 @@ public void longHBaseQualifierNameUsingLookupTable() throws Exception {
600600
*
601601
* @throws Exception if test fails to run
602602
*/
603-
@Test(groups = { "hbase", "features", "gpdb" })
603+
@Test(groups = { "hbase" })
604604
public void emptyHBaseTable() throws Exception {
605605

606606
HBaseTable emptyTable = new HBaseTable("empty_table", new String[] { "cf1" });

‎automation/src/test/java/org/apache/cloudberry/pxf/automation/smoke/HBaseSmokeTest.java‎

Lines changed: 0 additions & 99 deletions
This file was deleted.

‎automation/src/test/resources/sut/default.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
<hbase>
7373
<class>org.apache.cloudberry.pxf.automation.components.hbase.HBase</class>
7474
<host>localhost</host>
75+
<hbaseRoot>/home/gpadmin/workspace/singlecluster/hbase</hbaseRoot>
7576
</hbase>
7677

7778
<hive>

0 commit comments

Comments
 (0)