GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — [Enhancement] Support dry run mode for repairing cloud native tables … · APIJSON/starrocks@42352ad

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • fe
    • fe-core/src
      • main/java/com/starrocks
        • common/util
          • PropertyAnalyzer.java
        • lake
          • TabletRepairHelper.java
        • qe
          • DDLStmtExecutor.java
        • sql/analyzer
          • AdminStmtAnalyzer.java
      • test/java/com/starrocks/lake
        • TabletRepairHelperTest.java
    • fe-parser/src/main/java/com/starrocks/sql/ast
      • AdminRepairTableStmt.java
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java‎

Copy file name to clipboardExpand all lines: fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,8 @@ public class PropertyAnalyzer {
288288
public static final String PROPERTIES_ENFORCE_CONSISTENT_VERSION = "enforce_consistent_version";
289289
// Allows empty tablet recovery of tablets with no valid metadata
290290
public static final String PROPERTIES_ALLOW_EMPTY_TABLET_RECOVERY = "allow_empty_tablet_recovery";
291+
// If true, just return the repair plan without executing it
292+
public static final String PROPERTIES_DRY_RUN = "dry_run";
291293

292294
/**
293295
* Matches location labels like : ["*", "a:*", "bcd_123:*", "123bcd_:val_123", " a : b "],
Collapse file

‎fe/fe-core/src/main/java/com/starrocks/lake/TabletRepairHelper.java‎

Copy file name to clipboardExpand all lines: fe/fe-core/src/main/java/com/starrocks/lake/TabletRepairHelper.java
+125-18Lines changed: 125 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
import com.google.common.collect.Lists;
2020
import com.google.common.collect.Maps;
2121
import com.google.common.collect.Sets;
22+
import com.google.gson.Gson;
23+
import com.google.gson.annotations.SerializedName;
24+
import com.starrocks.catalog.Column;
2225
import com.starrocks.catalog.Database;
2326
import com.starrocks.catalog.MaterializedIndex;
2427
import com.starrocks.catalog.MaterializedIndex.IndexExtState;
@@ -43,13 +46,15 @@
4346
import com.starrocks.proto.TabletMetadataPB;
4447
import com.starrocks.proto.TabletMetadataRepairStatus;
4548
import com.starrocks.proto.TabletResult;
49+
import com.starrocks.qe.ShowResultSetMetaData;
4650
import com.starrocks.rpc.BrpcProxy;
4751
import com.starrocks.rpc.LakeService;
4852
import com.starrocks.rpc.RpcException;
4953
import com.starrocks.server.GlobalStateMgr;
5054
import com.starrocks.sql.ast.AdminRepairTableStmt;
5155
import com.starrocks.system.ComputeNode;
5256
import com.starrocks.thrift.TStatusCode;
57+
import com.starrocks.type.TypeFactory;
5358
import com.starrocks.warehouse.cngroup.ComputeResource;
5459
import org.apache.logging.log4j.LogManager;
5560
import org.apache.logging.log4j.Logger;
@@ -69,6 +74,18 @@ public class TabletRepairHelper {
6974
private static final long BATCH_VERSION_NUM = 5L;
7075
private static final String SST_FILE_SUFFIX = ".sst";
7176

77+
private static ShowResultSetMetaData DRY_RUN_REPAIR_RESULT_META_DATA;
78+
79+
static {
80+
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
81+
builder.addColumn(new Column("PartitionId", TypeFactory.createVarcharType(20)));
82+
builder.addColumn(new Column("VisibleVersion", TypeFactory.createVarcharType(20)));
83+
builder.addColumn(new Column("RepairStatus", TypeFactory.createVarcharType(60)));
84+
builder.addColumn(new Column("TabletRecoverInfo", TypeFactory.createVarcharType(65535)));
85+
builder.addColumn(new Column("ErrorMsg", TypeFactory.createVarcharType(65535)));
86+
DRY_RUN_REPAIR_RESULT_META_DATA = builder.build();
87+
}
88+
7289
// the version range [minVersion, maxVersion] is used to find valid tablet metadatas, both are included
7390
record PhysicalPartitionInfo(
7491
long physicalPartitionId,
@@ -585,24 +602,8 @@ private static Map<Long, String> repairTabletMetadata(PhysicalPartitionInfo info
585602
return tabletErrors;
586603
}
587604

588-
/**
589-
* Repairs the tablet metadata for a single physical partition.
590-
* This function attempts to find valid tablet metadata for all tablets within the given physical partition.
591-
* It iterates through versions to retrieve tablet metadatas from compute nodes by batches.
592-
* If valid metadata is found for all tablets, or
593-
* if `allowEmptyTabletRecovery` is true and empty metadata can be created for missing tablets,
594-
* it then sends these valid metadatas to the compute nodes to perform the repair operation.
595-
*
596-
* @param info Information about the physical partition.
597-
* @param enforceConsistentVersion Whether to enforce consistent metadata version across all tablets.
598-
* @param allowEmptyTabletRecovery Whether to allow creating empty metadata for tablets without valid metadata.
599-
* @param isFileBundling Whether file bundling is enabled for the table.
600-
* @return A map of tablet IDs to error messages for any tablets that failed to repair.
601-
* @throws Exception If an error occurs during the repair process.
602-
*/
603-
private static Map<Long, String> repairPhysicalPartition(PhysicalPartitionInfo info, boolean enforceConsistentVersion,
604-
boolean allowEmptyTabletRecovery, boolean isFileBundling)
605-
throws Exception {
605+
private static Map<Long, TabletMetadataPB> getValidTabletMetadatas(PhysicalPartitionInfo info,
606+
boolean enforceConsistentVersion) throws Exception {
606607
List<Long> allTablets = info.allTablets;
607608
Set<Long> unverifiedTablets = info.unverifiedTablets;
608609
long partitionMaxVersion = info.maxVersion;
@@ -626,6 +627,28 @@ private static Map<Long, String> repairPhysicalPartition(PhysicalPartitionInfo i
626627
break;
627628
}
628629
}
630+
return tabletToValidMetadata;
631+
}
632+
633+
/**
634+
* Repairs the tablet metadata for a single physical partition.
635+
* This function attempts to find valid tablet metadata for all tablets within the given physical partition.
636+
* It iterates through versions to retrieve tablet metadatas from compute nodes by batches.
637+
* If valid metadata is found for all tablets, or
638+
* if `allowEmptyTabletRecovery` is true and empty metadata can be created for missing tablets,
639+
* it then sends these valid metadatas to the compute nodes to perform the repair operation.
640+
*
641+
* @param info Information about the physical partition.
642+
* @param enforceConsistentVersion Whether to enforce consistent metadata version across all tablets.
643+
* @param allowEmptyTabletRecovery Whether to allow creating empty metadata for tablets without valid metadata.
644+
* @param isFileBundling Whether file bundling is enabled for the table.
645+
* @return A map of tablet IDs to error messages for any tablets that failed to repair.
646+
* @throws Exception If an error occurs during the repair process.
647+
*/
648+
private static Map<Long, String> repairPhysicalPartition(PhysicalPartitionInfo info, boolean enforceConsistentVersion,
649+
boolean allowEmptyTabletRecovery, boolean isFileBundling)
650+
throws Exception {
651+
Map<Long, TabletMetadataPB> tabletToValidMetadata = getValidTabletMetadatas(info, enforceConsistentVersion);
629652

630653
// check the valid tablet metadata, and create empty tablet metadata if no valid metadata is found
631654
checkOrCreateEmptyTabletMetadata(info, tabletToValidMetadata, enforceConsistentVersion, allowEmptyTabletRecovery);
@@ -706,4 +729,88 @@ public static void repair(AdminRepairTableStmt stmt, Database db, OlapTable tabl
706729
errorMsgsSize, errorMsgsSize > 1 ? "s" : "", Joiner.on(", ").join(errorMsgs)));
707730
}
708731
}
732+
733+
public static ShowResultSetMetaData getDryRunRepairResultMetaData() {
734+
return DRY_RUN_REPAIR_RESULT_META_DATA;
735+
}
736+
737+
private static class TabletRecoverInfo {
738+
@SerializedName("tabletId")
739+
public long tabletId;
740+
@SerializedName("recoverVersion")
741+
public long recoverVersion;
742+
743+
public TabletRecoverInfo(long tabletId, long recoverVersion) {
744+
this.tabletId = tabletId;
745+
this.recoverVersion = recoverVersion;
746+
}
747+
}
748+
749+
enum RepairStatus {
750+
NORMAL, // All tablets meta and data files are normal
751+
UNKNOWN, // Dry run repair exception
752+
RECOVERABLE, // Tablet has missing meta or data files, and is recoverable
753+
UNRECOVERABLE // Tablet has missing meta or data files, and is unrecoverable
754+
}
755+
756+
/**
757+
* Return the repair plan without executing it
758+
*/
759+
public static List<List<String>> dryRunRepair(AdminRepairTableStmt stmt, Database db, OlapTable table,
760+
List<String> partitionNames, ComputeResource computeResource)
761+
throws StarRocksException {
762+
boolean enforceConsistentVersion = stmt.isEnforceConsistentVersion();
763+
boolean allowEmptyTabletRecovery = stmt.isAllowEmptyTabletRecovery();
764+
Gson gson = new Gson();
765+
766+
// get physical partition ids in db table read lock
767+
List<Long> physicalPartitionIds = getPhysicalPartitionIds(db, table, partitionNames);
768+
769+
List<List<String>> result = Lists.newArrayList();
770+
for (Long physicalPartitionId : physicalPartitionIds) {
771+
long visibleVersion = -1;
772+
RepairStatus repairStatus = RepairStatus.UNKNOWN;
773+
List<TabletRecoverInfo> tabletRecoverInfos = Lists.newArrayList();
774+
String errorMsg = "";
775+
776+
try {
777+
PhysicalPartitionInfo info =
778+
getPhysicalPartitionInfo(db, table, physicalPartitionId, enforceConsistentVersion, computeResource);
779+
visibleVersion = info.maxVersion;
780+
781+
Map<Long, TabletMetadataPB> validMetadatas = getValidTabletMetadatas(info, enforceConsistentVersion);
782+
783+
// check the valid tablet metadata, and create empty tablet metadata if no valid metadata is found
784+
checkOrCreateEmptyTabletMetadata(info, validMetadatas, enforceConsistentVersion, allowEmptyTabletRecovery);
785+
786+
// physical partition is recoverable
787+
repairStatus = RepairStatus.RECOVERABLE;
788+
errorMsg = "";
789+
for (Long tabletId : info.allTablets) {
790+
TabletMetadataPB metadata = validMetadatas.get(tabletId);
791+
Preconditions.checkNotNull(metadata);
792+
long recoverVersion = metadata.version;
793+
tabletRecoverInfos.add(new TabletRecoverInfo(tabletId, recoverVersion));
794+
}
795+
} catch (AlreadyExistsException e) {
796+
repairStatus = RepairStatus.NORMAL;
797+
tabletRecoverInfos.clear();
798+
errorMsg = "";
799+
} catch (StarRocksException e) {
800+
repairStatus = RepairStatus.UNRECOVERABLE;
801+
tabletRecoverInfos.clear();
802+
errorMsg = e.getMessage();
803+
} catch (Exception e) {
804+
LOG.warn("Fail to dry run repair tablet metadata for partition {}", physicalPartitionId, e);
805+
repairStatus = RepairStatus.UNKNOWN;
806+
tabletRecoverInfos.clear();
807+
errorMsg = String.format("ERROR: %s", e.getMessage());
808+
}
809+
810+
result.add(Lists.newArrayList(String.valueOf(physicalPartitionId), String.valueOf(visibleVersion),
811+
repairStatus.name(), gson.toJson(tabletRecoverInfos), errorMsg));
812+
}
813+
814+
return result;
815+
}
709816
}
Collapse file

‎fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java‎

Copy file name to clipboardExpand all lines: fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java
+20-2Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -829,6 +829,9 @@ public ShowResultSet visitTruncateTableStatement(TruncateTableStmt stmt, Connect
829829

830830
@Override
831831
public ShowResultSet visitAdminRepairTableStatement(AdminRepairTableStmt stmt, ConnectContext context) {
832+
List<List<String>> result = Lists.newArrayList();
833+
boolean isDryRun = stmt.isDryRun();
834+
832835
ErrorReport.wrapWithRuntimeException(() -> {
833836
String dbName = stmt.getDbName() != null ? stmt.getDbName() : context.getDatabase();
834837
Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName);
@@ -847,16 +850,31 @@ public ShowResultSet visitAdminRepairTableStatement(AdminRepairTableStmt stmt, C
847850
PartitionRef partitionRef = stmt.getPartitionRef();
848851
List<String> partitionNames = partitionRef != null ? partitionRef.getPartitionNames() : Lists.newArrayList();
849852
ComputeResource computeResource = context.getCurrentComputeResource();
850-
TabletRepairHelper.repair(stmt, db, (OlapTable) table, partitionNames, computeResource);
853+
854+
if (isDryRun) {
855+
List<List<String>> rows =
856+
TabletRepairHelper.dryRunRepair(stmt, db, (OlapTable) table, partitionNames, computeResource);
857+
result.addAll(rows);
858+
} else {
859+
TabletRepairHelper.repair(stmt, db, (OlapTable) table, partitionNames, computeResource);
860+
}
851861
} else if (table.isCloudNativeMaterializedView()) {
852862
// cloud native mv
853863
throw new DdlException("Repair cloud native materialized view is not supported");
854864
} else {
855865
// olap table or mv
866+
if (isDryRun) {
867+
throw new DdlException("Dry run is only supported for cloud-native tables");
868+
}
856869
GlobalStateMgr.getCurrentState().getTabletChecker().repairTable(context, stmt);
857870
}
858871
});
859-
return null;
872+
873+
if (isDryRun) {
874+
return new ShowResultSet(TabletRepairHelper.getDryRunRepairResultMetaData(), result);
875+
} else {
876+
return null;
877+
}
860878
}
861879

862880
@Override
Collapse file

‎fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AdminStmtAnalyzer.java‎

Copy file name to clipboardExpand all lines: fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AdminStmtAnalyzer.java
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,9 @@ public Void visitAdminRepairTableStatement(AdminRepairTableStmt adminRepairTable
175175
properties, PropertyAnalyzer.PROPERTIES_ALLOW_EMPTY_TABLET_RECOVERY, false);
176176
adminRepairTableStmt.setAllowEmptyTabletRecovery(allowEmptyTabletRecovery);
177177

178+
boolean dryRun = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_DRY_RUN, false);
179+
adminRepairTableStmt.setDryRun(dryRun);
180+
178181
if (!properties.isEmpty()) {
179182
ErrorReport.reportSemanticException(ErrorCode.ERR_UNKNOWN_PROPERTY, properties);
180183
}

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.