Skip to content

Commit 80b749f

Browse files
gengjun-gitgengjun-git
authored
[Refactor] Transform SessionVariable/Catalog/SecurityIntegration/StorageVolume/Dictionary related edit logs to WAL format (StarRocks#66207)
Signed-off-by: gengjun-git <gengjun@starrocks.com>
1 parent 1646b03 commit 80b749f

30 files changed

Lines changed: 3517 additions & 609 deletions

‎fe/fe-core/src/main/java/com/starrocks/authentication/AuthenticationMgr.java‎

Lines changed: 46 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.starrocks.persist.EditLog;
2626
import com.starrocks.persist.GroupProviderLog;
2727
import com.starrocks.persist.ImageWriter;
28+
import com.starrocks.persist.SecurityIntegrationPersistInfo;
2829
import com.starrocks.persist.metablock.MapEntryConsumer;
2930
import com.starrocks.persist.metablock.SRMetaBlockEOFException;
3031
import com.starrocks.persist.metablock.SRMetaBlockException;
@@ -532,24 +533,24 @@ public UserIdentity getUserIdentityByName(String userName) {
532533
//=========================================== Security Integration ==================================================
533534

534535
public void createSecurityIntegration(String name,
535-
Map<String, String> propertyMap,
536-
boolean isReplay) throws DdlException {
537-
SecurityIntegration securityIntegration;
538-
securityIntegration = SecurityIntegrationFactory.createSecurityIntegration(name, propertyMap);
539-
// atomic op
540-
SecurityIntegration result = nameToSecurityIntegrationMap.putIfAbsent(name, securityIntegration);
541-
if (result != null) {
536+
Map<String, String> propertyMap) throws DdlException {
537+
if (nameToSecurityIntegrationMap.containsKey(name)) {
542538
throw new DdlException("security integration '" + name + "' already exists");
543539
}
544-
if (!isReplay) {
545-
EditLog editLog = GlobalStateMgr.getCurrentState().getEditLog();
546-
editLog.logCreateSecurityIntegration(name, propertyMap);
547-
LOG.info("finished to create security integration '{}'", securityIntegration.toString());
548-
}
540+
SecurityIntegration securityIntegration = SecurityIntegrationFactory.createSecurityIntegration(name, propertyMap);
541+
EditLog editLog = GlobalStateMgr.getCurrentState().getEditLog();
542+
editLog.logCreateSecurityIntegration(new SecurityIntegrationPersistInfo(name, propertyMap), wal -> {
543+
nameToSecurityIntegrationMap.put(name, securityIntegration);
544+
});
545+
LOG.info("finished to create security integration '{}'", securityIntegration.toString());
546+
}
547+
548+
public void replayCreateSecurityIntegration(String name, Map<String, String> propertyMap) {
549+
SecurityIntegration securityIntegration = SecurityIntegrationFactory.createSecurityIntegration(name, propertyMap);
550+
nameToSecurityIntegrationMap.put(name, securityIntegration);
549551
}
550552

551-
public void alterSecurityIntegration(String name, Map<String, String> alterProps,
552-
boolean isReplay) throws DdlException {
553+
public void alterSecurityIntegration(String name, Map<String, String> alterProps) throws DdlException {
553554
SecurityIntegration securityIntegration = nameToSecurityIntegrationMap.get(name);
554555
if (securityIntegration == null) {
555556
throw new DdlException("security integration '" + name + "' not found");
@@ -558,30 +559,28 @@ public void alterSecurityIntegration(String name, Map<String, String> alterProps
558559
Map<String, String> newProps = Maps.newHashMap(securityIntegration.getPropertyMap());
559560
// update props
560561
newProps.putAll(alterProps);
561-
SecurityIntegration newSecurityIntegration;
562-
newSecurityIntegration = SecurityIntegrationFactory.createSecurityIntegration(name, newProps);
563-
// update map
564-
nameToSecurityIntegrationMap.put(name, newSecurityIntegration);
565-
if (!isReplay) {
566-
EditLog editLog = GlobalStateMgr.getCurrentState().getEditLog();
567-
editLog.logAlterSecurityIntegration(name, alterProps);
568-
LOG.info("finished to alter security integration '{}' with updated properties {}",
569-
name, alterProps);
570-
}
562+
SecurityIntegration newSecurityIntegration = SecurityIntegrationFactory.createSecurityIntegration(name, newProps);
563+
EditLog editLog = GlobalStateMgr.getCurrentState().getEditLog();
564+
editLog.logAlterSecurityIntegration(new SecurityIntegrationPersistInfo(name, alterProps), wal -> {
565+
// update map
566+
nameToSecurityIntegrationMap.put(name, newSecurityIntegration);
567+
});
568+
LOG.info("finished to alter security integration '{}' with updated properties {}", name, alterProps);
571569
}
572570
}
573571

574-
public void dropSecurityIntegration(String name, boolean isReplay) throws DdlException {
572+
573+
574+
public void dropSecurityIntegration(String name) throws DdlException {
575575
if (!nameToSecurityIntegrationMap.containsKey(name)) {
576576
throw new DdlException("security integration '" + name + "' not found");
577577
}
578578

579-
SecurityIntegration result = nameToSecurityIntegrationMap.remove(name);
580-
if (!isReplay && result != null) {
581-
EditLog editLog = GlobalStateMgr.getCurrentState().getEditLog();
582-
editLog.logDropSecurityIntegration(name);
583-
LOG.info("finished to drop security integration '{}'", name);
584-
}
579+
EditLog editLog = GlobalStateMgr.getCurrentState().getEditLog();
580+
editLog.logDropSecurityIntegration(new SecurityIntegrationPersistInfo(name, null), wal -> {
581+
nameToSecurityIntegrationMap.remove(name);
582+
});
583+
LOG.info("finished to drop security integration '{}'", name);
585584
}
586585

587586
public SecurityIntegration getSecurityIntegration(String name) {
@@ -592,19 +591,24 @@ public Set<SecurityIntegration> getAllSecurityIntegrations() {
592591
return new HashSet<>(nameToSecurityIntegrationMap.values());
593592
}
594593

595-
public void replayCreateSecurityIntegration(String name, Map<String, String> propertyMap)
596-
throws DdlException {
597-
createSecurityIntegration(name, propertyMap, true);
598-
}
599-
600-
public void replayAlterSecurityIntegration(String name, Map<String, String> alterProps)
601-
throws DdlException {
602-
alterSecurityIntegration(name, alterProps, true);
594+
public void replayAlterSecurityIntegration(String name, Map<String, String> alterProps) {
595+
SecurityIntegration securityIntegration = nameToSecurityIntegrationMap.get(name);
596+
if (securityIntegration != null) {
597+
// COW
598+
Map<String, String> newProps = Maps.newHashMap(securityIntegration.getPropertyMap());
599+
// update props
600+
newProps.putAll(alterProps);
601+
SecurityIntegration newSecurityIntegration =
602+
SecurityIntegrationFactory.createSecurityIntegration(name, newProps);
603+
// update map
604+
nameToSecurityIntegrationMap.put(name, newSecurityIntegration);
605+
LOG.info("finished to replay alter security integration '{}' with updated properties {}",
606+
name, alterProps);
607+
}
603608
}
604609

605-
public void replayDropSecurityIntegration(String name)
606-
throws DdlException {
607-
dropSecurityIntegration(name, true);
610+
public void replayDropSecurityIntegration(String name) throws DdlException {
611+
nameToSecurityIntegrationMap.remove(name);
608612
}
609613

610614
// ---------------------------------------- Group Provider Statement --------------------------------------

‎fe/fe-core/src/main/java/com/starrocks/catalog/Dictionary.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,11 @@ public long getNextSchedulableTime() {
152152
return nextSchedulableTime.get();
153153
}
154154

155+
// Only for test
156+
public void setNextSchedulableTime(long nextSchedulableTime) {
157+
this.nextSchedulableTime.set(nextSchedulableTime);
158+
}
159+
155160
public void updateNextSchedulableTime(long refreshInterval) {
156161
if (refreshInterval > 0) {
157162
nextSchedulableTime.set(System.currentTimeMillis() + refreshInterval);
@@ -286,7 +291,6 @@ public void buildDictionaryProperties() throws DdlException {
286291
throw new DdlException("unknown property for dictionary: " + key);
287292
}
288293
}
289-
return;
290294
}
291295

292296
public String buildQuery() {
@@ -319,10 +323,11 @@ public synchronized void resetState() {
319323
this.setLastSuccessVersion(0);
320324
}
321325

322-
public synchronized void setRefreshing() {
326+
public synchronized void setRefreshing(long ts) {
323327
this.stateBeforeRefresh = this.state;
324328
this.state = DictionaryState.REFRESHING;
325-
this.lastSuccessRefreshTime = System.currentTimeMillis();
329+
this.lastSuccessRefreshTime = ts;
330+
this.nextSchedulableTime.set(ts + refreshInterval);
326331
this.setErrorMsg("");
327332
}
328333

@@ -331,10 +336,11 @@ public synchronized void setCommitting() {
331336
this.stateBeforeRefresh = null;
332337
}
333338

334-
public synchronized void setFinished() {
339+
public synchronized void setFinished(long ts, long version) {
335340
this.state = DictionaryState.FINISHED;
336-
this.lastSuccessFinishedTime = System.currentTimeMillis();
341+
this.lastSuccessFinishedTime = ts;
337342
this.stateBeforeRefresh = null;
343+
this.lastSuccessVersion = version;
338344
}
339345

340346
public synchronized void setCancelled() {
@@ -369,6 +375,10 @@ public synchronized void resetStateBeforeRefresh() {
369375
this.stateBeforeRefresh = null;
370376
}
371377
}
378+
379+
protected String getRuntimeErrMsg() {
380+
return runtimeErrMsg;
381+
}
372382

373383
public List<String> getInfo() {
374384
List<String> info = new ArrayList<>();

0 commit comments

Comments
 (0)