diff --git a/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreProperties.java b/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionAutoConfigProperties.java similarity index 53% rename from starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreProperties.java rename to starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionAutoConfigProperties.java index 41ab68f0f..f447891f1 100644 --- a/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreProperties.java +++ b/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionAutoConfigProperties.java @@ -7,15 +7,27 @@ @Getter @Setter @ConfigurationProperties(prefix = "store", ignoreUnknownFields = true) -public class TransactionStoreProperties { +public class TransactionAutoConfigProperties { private Transaction transaction; @Getter @Setter public static final class Transaction { - private boolean enabled = true; - private boolean apiEnabled = true; - private Endpoints endpoints = new Endpoints(); + private boolean enabled = true; + private boolean apiEnabled = true; + private Endpoints endpoints = new Endpoints(); + /** + * Enable pruning of Transaction + */ + private boolean pruningEnabled = false; + /** + * Transaction Pruning interval in minutes + */ + private int pruningInterval = 1440; //In minutes + /** + * safe slot count to keep before pruning the Transaction + */ + private int pruningSafeSlot = 43200; // 2160 blocks } @Getter diff --git a/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreAutoConfiguration.java b/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreAutoConfiguration.java index 3018f3b58..f2f9198d8 100644 --- a/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreAutoConfiguration.java +++ b/starters/transaction-spring-boot-starter/src/main/java/com/bloxbean/cardano/yaci/store/starter/transaction/TransactionStoreAutoConfiguration.java @@ -2,18 +2,31 @@ import com.bloxbean.cardano.yaci.store.api.transaction.TransactionApiConfiguration; import com.bloxbean.cardano.yaci.store.transaction.TransactionStoreConfiguration; +import com.bloxbean.cardano.yaci.store.transaction.TransactionStoreProperties; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; @AutoConfiguration -@EnableConfigurationProperties(TransactionStoreProperties.class) +@EnableConfigurationProperties(TransactionAutoConfigProperties.class) @Import({TransactionStoreConfiguration.class, TransactionApiConfiguration.class}) @Slf4j public class TransactionStoreAutoConfiguration { @Autowired - TransactionStoreProperties properties; + TransactionAutoConfigProperties properties; + + @Bean + public TransactionStoreProperties transactionStoreProperties() { + var transactionStoreProperties = new TransactionStoreProperties(); + + transactionStoreProperties.setPruningEnabled(properties.getTransaction().isPruningEnabled()); + transactionStoreProperties.setPruningInterval(properties.getTransaction().getPruningInterval()); + transactionStoreProperties.setPruningSafeSlot(properties.getTransaction().getPruningSafeSlot()); + + return transactionStoreProperties; + } } diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreConfiguration.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreConfiguration.java index ff1e3b45f..1ae75447b 100644 --- a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreConfiguration.java +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreConfiguration.java @@ -43,7 +43,7 @@ public TransactionStorage transactionStorage(TxnEntityRepository txnEntityReposi @Bean @ConditionalOnMissingBean public TransactionWitnessStorage transactionWitnessStorage(TxnWitnessRepository txnWitnessRepository, TxnMapper txnMapper) { - return new TransactionWitnessStorageImpl(txnWitnessRepository, txnMapper); + return new TransactionWitnessStorageImpl(txnWitnessRepository, txnMapper, dslContext); } @Bean diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreProperties.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreProperties.java new file mode 100644 index 000000000..349f9a70f --- /dev/null +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/TransactionStoreProperties.java @@ -0,0 +1,22 @@ +package com.bloxbean.cardano.yaci.store.transaction; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class TransactionStoreProperties { + + @Builder.Default + private boolean pruningEnabled = false; + + @Builder.Default + private int pruningInterval = 1440; //In minutes + + @Builder.Default + private int pruningSafeSlot = 43200; // 2160 blocks +} diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/scheduler/TransactionPruningService.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/scheduler/TransactionPruningService.java new file mode 100644 index 000000000..0d4dde70b --- /dev/null +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/scheduler/TransactionPruningService.java @@ -0,0 +1,84 @@ +package com.bloxbean.cardano.yaci.store.transaction.scheduler; + + +import com.bloxbean.cardano.yaci.store.common.service.CursorService; +import com.bloxbean.cardano.yaci.store.events.EpochChangeEvent; +import com.bloxbean.cardano.yaci.store.transaction.TransactionStoreProperties; +import com.bloxbean.cardano.yaci.store.transaction.storage.TransactionStorage; +import com.bloxbean.cardano.yaci.store.transaction.storage.TransactionWitnessStorage; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@Component +@ConditionalOnProperty( + value = "store.transaction.pruning-enabled", + havingValue = "true" +) +@RequiredArgsConstructor +@Slf4j +public class TransactionPruningService { + private final TransactionStorage transactionStorage; + private final TransactionWitnessStorage transactionWitnessStorage; + private final CursorService cursorService; + private final TransactionStoreProperties transactionStoreProperties; + + private final AtomicBoolean isPruning = new AtomicBoolean(false); + + @PostConstruct + public void init() { + log.info("<< Transaction Pruning Service Enabled >>"); + } + + @Scheduled(fixedRateString = "${store.transaction.pruning-interval:1440}", timeUnit = TimeUnit.MINUTES) + public void handleTransactionPruning() { + if (isPruning.get()) { + log.info("Transaction pruning is already in progress. Skipping this run !!!"); + return; + } + + Thread.startVirtualThread(this::deleteOldTransactions); + } + + @EventListener + @Transactional + public void handleEpochChangeEvent(EpochChangeEvent epochChangeEvent) { + if (isPruning.get()) { + log.info("Transaction pruning is already in progress. Skipping this run !!!"); + return; + } + + Thread.startVirtualThread(this::deleteOldTransactions); + } + + private void deleteOldTransactions() { + isPruning.set(true); + try { + cursorService.getCursor().ifPresent(cursor -> { + log.info("Current cursor: {}", cursor.getBlock()); + + var slot = cursor.getSlot() - transactionStoreProperties.getPruningSafeSlot(); + if (slot > 0) { + long t1 = System.currentTimeMillis(); + var deleteTxCount = + transactionStorage.deleteBySlotLessThan(slot); + var deleteTxWitnessCount = transactionWitnessStorage.deleteBySlotLessThan(slot); + // skip pruning invalid_transaction and withdrawn because these 2 tables do not have too much data + long t2 = System.currentTimeMillis(); + log.info("Deleted {} transactions and {} transaction witnesses before slot {}, Time taken: {} ms", + deleteTxCount, deleteTxWitnessCount, slot, (t2 - t1)); + } + }); + } finally { + isPruning.set(false); + } + } +} diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionStorage.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionStorage.java index 659d91f4e..f2e524a56 100644 --- a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionStorage.java +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionStorage.java @@ -7,4 +7,5 @@ public interface TransactionStorage { void saveAll(List txList); int deleteBySlotGreaterThan(long slot); + int deleteBySlotLessThan(long slot); } diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionWitnessStorage.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionWitnessStorage.java index 8b13faef6..4463f48f7 100644 --- a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionWitnessStorage.java +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/TransactionWitnessStorage.java @@ -7,4 +7,5 @@ public interface TransactionWitnessStorage { void saveAll(List txnWitnesses); int deleteBySlotGreaterThan(long slot); + int deleteBySlotLessThan(long slot); } diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionStorageImpl.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionStorageImpl.java index 9b9c145f6..7ca8f0822 100644 --- a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionStorageImpl.java +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionStorageImpl.java @@ -8,10 +8,13 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.stream.Collectors; +import static com.bloxbean.cardano.yaci.store.transaction.jooq.Tables.TRANSACTION; + @RequiredArgsConstructor @Slf4j public class TransactionStorageImpl implements TransactionStorage { @@ -29,4 +32,10 @@ public void saveAll(List txnList) { public int deleteBySlotGreaterThan(long slot) { return txnEntityRepository.deleteBySlotGreaterThan(slot); } + + @Override + @Transactional + public int deleteBySlotLessThan(long slot) { + return dsl.deleteFrom(TRANSACTION).where(TRANSACTION.SLOT.lessThan(slot)).execute(); + } } diff --git a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionWitnessStorageImpl.java b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionWitnessStorageImpl.java index 3449508bf..e8a3f821d 100644 --- a/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionWitnessStorageImpl.java +++ b/stores/transaction/src/main/java/com/bloxbean/cardano/yaci/store/transaction/storage/impl/TransactionWitnessStorageImpl.java @@ -7,14 +7,18 @@ import com.bloxbean.cardano.yaci.store.transaction.storage.impl.repository.TxnWitnessRepository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; import java.util.List; +import static com.bloxbean.cardano.yaci.store.transaction.jooq.Tables.TRANSACTION_WITNESS; + @RequiredArgsConstructor @Slf4j public class TransactionWitnessStorageImpl implements TransactionWitnessStorage { private final TxnWitnessRepository txnWitnessRepository; private final TxnMapper mapper; + private final DSLContext dsl; @Override public void saveAll(List txnWitnesses) { @@ -26,4 +30,9 @@ public void saveAll(List txnWitnesses) { public int deleteBySlotGreaterThan(long slot) { return txnWitnessRepository.deleteBySlotGreaterThan(slot); } + + @Override + public int deleteBySlotLessThan(long slot) { + return dsl.deleteFrom(TRANSACTION_WITNESS).where(TRANSACTION_WITNESS.SLOT.lessThan(slot)).execute(); + } }