Skip to content

Commit e1c8820

Browse files
Kota-SHsharmaar12kgeiszshankotkabhishek4
committed
HBASE-29081: Add HBase Read Replica Cluster feature (#8044)
(cherry picked from commit 9ba5d3e) Signed-off-by: Tak Lon (Stephen) Wu <taklwu@apache.org> Signed-off-by: Charles Connell <charles@charlesconnell.com> Co-authored-by: Anuj Sharma <anuj.sharma@cloudera.com> Co-authored-by: Kevin Geiszler <kgeiszler@cloudera.com> Co-authored-by: Shanmukha Haripriya Kota <skota@cloudera.com> Co-authored-by: Abhishek Kothalikar <kabhishek@cloudera.com> Co-authored-by: Andor Molnar <andor@apache.org>
1 parent e1b8801 commit e1c8820

111 files changed

Lines changed: 7963 additions & 249 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.hadoop.fs.Path;
2929
import org.apache.hadoop.fs.PathFilter;
3030
import org.apache.hadoop.hbase.HConstants;
31+
import org.apache.hadoop.hbase.TableName;
3132
import org.apache.hadoop.hbase.backup.util.BackupUtils;
3233
import org.apache.hadoop.hbase.client.Connection;
3334
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
@@ -156,7 +157,7 @@ private List<String> getLogFilesForNewBackup(Map<String, Long> olderTimestamps,
156157
LOG.debug("currentLogFile: " + log.getPath().toString());
157158
if (AbstractFSWALProvider.isMetaFile(log.getPath())) {
158159
if (LOG.isDebugEnabled()) {
159-
LOG.debug("Skip hbase:meta log file: " + log.getPath().getName());
160+
LOG.debug("Skip {} log file: {}", TableName.META_TABLE_NAME, log.getPath().getName());
160161
}
161162
continue;
162163
}

‎hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,6 @@ private static boolean isHMasterWAL(Path path) {
204204
String fn = path.getName();
205205
return fn.startsWith(WALProcedureStore.LOG_PREFIX)
206206
|| fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX)
207-
|| path.toString().contains("/%s/".formatted(MasterRegionFactory.MASTER_STORE_DIR));
207+
|| path.toString().contains("/%s/".formatted(MasterRegionFactory.MASTER_REGION_DIR_NAME));
208208
}
209209
}

‎hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,8 @@ private void processMetaRecord(Result result) throws IOException {
170170
* Initialize the region assignment snapshot by scanning the hbase:meta table
171171
*/
172172
public void initialize() throws IOException {
173-
LOG.info("Start to scan the hbase:meta for the current region assignment " + "snappshot");
173+
LOG.info("Start to scan {} for the current region assignment snapshot",
174+
TableName.META_TABLE_NAME);
174175
// Scan hbase:meta to pick up user regions
175176
try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME);
176177
ResultScanner scanner = metaTable.getScanner(HConstants.CATALOG_FAMILY)) {
@@ -187,7 +188,8 @@ public void initialize() throws IOException {
187188
}
188189
}
189190
}
190-
LOG.info("Finished to scan the hbase:meta for the current region assignment" + "snapshot");
191+
LOG.info("Finished scanning {} for the current region assignment snapshot",
192+
TableName.META_TABLE_NAME);
191193
}
192194

193195
private void addRegion(RegionInfo regionInfo) {

‎hbase-client/src/main/java/org/apache/hadoop/hbase/ActiveClusterSuffix.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase;
19+
20+
import java.io.IOException;
21+
import java.util.Objects;
22+
import org.apache.hadoop.conf.Configuration;
23+
import org.apache.hadoop.hbase.exceptions.DeserializationException;
24+
import org.apache.hadoop.hbase.util.Bytes;
25+
import org.apache.yetus.audience.InterfaceAudience;
26+
27+
import org.apache.hbase.thirdparty.com.google.common.base.Strings;
28+
29+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
30+
import org.apache.hadoop.hbase.shaded.protobuf.generated.ActiveClusterSuffixProtos;
31+
32+
/**
33+
* The read-replica cluster id for this cluster. It is serialized to the filesystem and up into
34+
* zookeeper. This is a container for the id. Also knows how to serialize and deserialize the
35+
* cluster id.
36+
*/
37+
@InterfaceAudience.Private
38+
public class ActiveClusterSuffix implements ClusterIdFile {
39+
private final String clusterId;
40+
private final String suffix;
41+
42+
public static class Parser implements ClusterIdFileParser<ActiveClusterSuffix> {
43+
44+
@Override
45+
public String getFileName() {
46+
return HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME;
47+
}
48+
49+
/**
50+
* Parse the serialized representation of the {@link ActiveClusterSuffix}
51+
* @param bytes A pb serialized {@link ActiveClusterSuffix} instance with pb magic prefix
52+
* @return An instance of {@link ActiveClusterSuffix} made from <code>bytes</code>
53+
* @see #toByteArray()
54+
*/
55+
@Override
56+
public ActiveClusterSuffix parseFrom(byte[] bytes) throws DeserializationException {
57+
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
58+
int pblen = ProtobufUtil.lengthOfPBMagic();
59+
ActiveClusterSuffixProtos.ActiveClusterSuffix.Builder builder =
60+
ActiveClusterSuffixProtos.ActiveClusterSuffix.newBuilder();
61+
ActiveClusterSuffixProtos.ActiveClusterSuffix cs = null;
62+
try {
63+
ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
64+
cs = builder.build();
65+
} catch (IOException e) {
66+
throw new DeserializationException(e);
67+
}
68+
return convert(cs);
69+
} else {
70+
// Presume it was written out this way, the old way.
71+
return new ActiveClusterSuffix(Bytes.toString(bytes));
72+
}
73+
}
74+
75+
@Override
76+
public ActiveClusterSuffix readString(String input) {
77+
return new ActiveClusterSuffix(input);
78+
}
79+
}
80+
81+
public ActiveClusterSuffix(final String clusterId, final String suffix) {
82+
this.clusterId = clusterId;
83+
this.suffix = suffix;
84+
}
85+
86+
public ActiveClusterSuffix(final String input) {
87+
String[] parts = input.split(":", 2);
88+
this.clusterId = parts[0];
89+
if (parts.length > 1) {
90+
this.suffix = parts[1];
91+
} else {
92+
this.suffix = "";
93+
}
94+
}
95+
96+
public static ActiveClusterSuffix parseFrom(byte[] bytes) throws DeserializationException {
97+
return new Parser().parseFrom(bytes);
98+
}
99+
100+
public static ActiveClusterSuffix fromConfig(Configuration conf, ClusterId clusterId) {
101+
return new ActiveClusterSuffix(clusterId.toString(), conf
102+
.get(HConstants.HBASE_META_TABLE_SUFFIX, HConstants.HBASE_META_TABLE_SUFFIX_DEFAULT_VALUE));
103+
}
104+
105+
/** Returns The active cluster suffix serialized using pb w/ pb magic prefix */
106+
public byte[] toByteArray() {
107+
return ProtobufUtil.prependPBMagic(convert().toByteArray());
108+
}
109+
110+
/** Returns A pb instance to represent this instance. */
111+
public ActiveClusterSuffixProtos.ActiveClusterSuffix convert() {
112+
return ActiveClusterSuffixProtos.ActiveClusterSuffix.newBuilder().setClusterId(clusterId)
113+
.setSuffix(suffix).build();
114+
}
115+
116+
/** Returns A {@link ActiveClusterSuffix} made from the passed in <code>cs</code> */
117+
public static ActiveClusterSuffix
118+
convert(final ActiveClusterSuffixProtos.ActiveClusterSuffix cs) {
119+
return new ActiveClusterSuffix(cs.getClusterId(), cs.getSuffix());
120+
}
121+
122+
/**
123+
* @see java.lang.Object#toString()
124+
*/
125+
@Override
126+
public String toString() {
127+
return String.format("%s:%s", this.clusterId,
128+
Strings.isNullOrEmpty(this.suffix) ? "<blank>" : this.suffix);
129+
}
130+
131+
@Override
132+
public boolean equals(Object o) {
133+
if (o == null || getClass() != o.getClass()) return false;
134+
ActiveClusterSuffix that = (ActiveClusterSuffix) o;
135+
return Objects.equals(clusterId, that.clusterId) && Objects.equals(suffix, that.suffix);
136+
}
137+
138+
@Override
139+
public int hashCode() {
140+
return Objects.hash(clusterId, suffix);
141+
}
142+
}

‎hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java‎

Lines changed: 39 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,47 @@
3131
* is a container for the id. Also knows how to serialize and deserialize the cluster id.
3232
*/
3333
@InterfaceAudience.Private
34-
public class ClusterId {
34+
public class ClusterId implements ClusterIdFile {
3535
private final String id;
3636

37+
public static class Parser implements ClusterIdFileParser<ClusterId> {
38+
39+
@Override
40+
public String getFileName() {
41+
return HConstants.CLUSTER_ID_FILE_NAME;
42+
}
43+
44+
/**
45+
* Parse the serialized representation of the {@link ClusterId}
46+
* @param bytes A pb serialized {@link ClusterId} instance with pb magic prefix
47+
* @return An instance of {@link ClusterId} made from <code>bytes</code>
48+
* @see #toByteArray()
49+
*/
50+
@Override
51+
public ClusterId parseFrom(byte[] bytes) throws DeserializationException {
52+
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
53+
int pblen = ProtobufUtil.lengthOfPBMagic();
54+
ClusterIdProtos.ClusterId.Builder builder = ClusterIdProtos.ClusterId.newBuilder();
55+
ClusterIdProtos.ClusterId cid = null;
56+
try {
57+
ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
58+
cid = builder.build();
59+
} catch (IOException e) {
60+
throw new DeserializationException(e);
61+
}
62+
return convert(cid);
63+
} else {
64+
// Presume it was written out this way, the old way.
65+
return new ClusterId(Bytes.toString(bytes));
66+
}
67+
}
68+
69+
@Override
70+
public ClusterId readString(String input) {
71+
return new ClusterId(input);
72+
}
73+
}
74+
3775
/**
3876
* New ClusterID. Generates a uniqueid.
3977
*/
@@ -50,30 +88,6 @@ public byte[] toByteArray() {
5088
return ProtobufUtil.prependPBMagic(convert().toByteArray());
5189
}
5290

53-
/**
54-
* Parse the serialized representation of the {@link ClusterId}
55-
* @param bytes A pb serialized {@link ClusterId} instance with pb magic prefix
56-
* @return An instance of {@link ClusterId} made from <code>bytes</code>
57-
* @see #toByteArray()
58-
*/
59-
public static ClusterId parseFrom(final byte[] bytes) throws DeserializationException {
60-
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
61-
int pblen = ProtobufUtil.lengthOfPBMagic();
62-
ClusterIdProtos.ClusterId.Builder builder = ClusterIdProtos.ClusterId.newBuilder();
63-
ClusterIdProtos.ClusterId cid = null;
64-
try {
65-
ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
66-
cid = builder.build();
67-
} catch (IOException e) {
68-
throw new DeserializationException(e);
69-
}
70-
return convert(cid);
71-
} else {
72-
// Presume it was written out this way, the old way.
73-
return new ClusterId(Bytes.toString(bytes));
74-
}
75-
}
76-
7791
/** Returns A pb instance to represent this instance. */
7892
public ClusterIdProtos.ClusterId convert() {
7993
ClusterIdProtos.ClusterId.Builder builder = ClusterIdProtos.ClusterId.newBuilder();

‎hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterIdFile.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase;
19+
20+
import org.apache.yetus.audience.InterfaceAudience;
21+
22+
/**
23+
* Represents a cluster identification file on the master file system. e.g. Cluster ID = hbase.id
24+
* Active read-replica cluster ID = active.cluster.suffix.id
25+
*/
26+
@InterfaceAudience.Private
27+
public interface ClusterIdFile {
28+
29+
/**
30+
* Return file contents in a byte array.
31+
*/
32+
byte[] toByteArray();
33+
34+
}

‎hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterIdFileParser.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase;
19+
20+
import org.apache.hadoop.hbase.exceptions.DeserializationException;
21+
import org.apache.yetus.audience.InterfaceAudience;
22+
23+
/**
24+
* Generic parser interface for Cluster Id files.
25+
* @see ClusterIdFile
26+
*/
27+
@InterfaceAudience.Private
28+
public interface ClusterIdFileParser<T> {
29+
30+
/**
31+
* Get default file name of cluster id file.
32+
*/
33+
String getFileName();
34+
35+
/**
36+
* Parse cluster id data from byte representation.
37+
* @param bytes the protobuf data
38+
* @return the cluster id data object
39+
*/
40+
T parseFrom(final byte[] bytes) throws DeserializationException;
41+
42+
/**
43+
* Parser cluster id data from String representation.
44+
* @param input the input string
45+
* @return the cluster id data object
46+
*/
47+
T readString(String input);
48+
}

0 commit comments

Comments
 (0)