Skip to content

Commit 5962179

Browse files
cbeach47cbeach47
authored
Feature/20 change caching (openequella#27)
* openequella#20 enabled caching and checking atts per item
1 parent 7a1b286 commit 5962179

File tree

4 files changed

+159
-53
lines changed

4 files changed

+159
-53
lines changed

‎src/main/java/org/apereo/openequella/tools/toolbox/CheckFilesDriver.java

+1
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ public static boolean run() {
159159
return false;
160160
}
161161
case DB_BATCH_ITEMS_PER_ITEM_ATTS:
162+
case DB_BATCH_ITEMS_PER_ITEM_ATTS_CONFIRM_INLINE:
162163
case DB_ALL_ITEMS_ALL_ATTS:
163164
{
164165
return (new CheckFilesDbHandler()).execute();

‎src/main/java/org/apereo/openequella/tools/toolbox/Config.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,11 @@ public static enum ToolboxFunction {
144144
public enum CheckFilesType {
145145
REST, // Implemented, but needs testing after the open source effort
146146
DB_ALL_ITEMS_ALL_ATTS, // Currently the only supported method
147-
DB_BATCH_ITEMS_PER_ITEM_ATTS // Implemented, but needs testing after the open source effort
147+
DB_BATCH_ITEMS_PER_ITEM_ATTS,
148+
// supports caching by a block of items, and for each item cached,
149+
// pulls the attachments for the items and checks existance BEFORE
150+
// getting more items.
151+
DB_BATCH_ITEMS_PER_ITEM_ATTS_CONFIRM_INLINE
148152
}
149153

150154
public enum CheckFilesSupportedDB {
@@ -352,6 +356,11 @@ public void checkConfigsCheckFiles() {
352356
}
353357
case DB_BATCH_ITEMS_PER_ITEM_ATTS:
354358
{
359+
// pass through
360+
}
361+
case DB_BATCH_ITEMS_PER_ITEM_ATTS_CONFIRM_INLINE:
362+
{
363+
// pass through
355364
}
356365
case DB_ALL_ITEMS_ALL_ATTS:
357366
{

‎src/main/java/org/apereo/openequella/tools/toolbox/checkFiles/CheckFilesDbHandler.java

+93-52
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apereo.openequella.tools.toolbox.Config;
3636
import org.apereo.openequella.tools.toolbox.utils.Check;
3737
import org.apereo.openequella.tools.toolbox.utils.CheckFilesUtils;
38+
import org.apereo.openequella.tools.toolbox.utils.Pair;
3839
import org.apereo.openequella.tools.toolbox.utils.WhereClauseExpression;
3940

4041
/**
@@ -104,39 +105,47 @@ public boolean execute() {
104105
return false;
105106
}
106107

107-
// Cache Items
108-
if (Config.get(Config.CF_MODE).contains("ALL_ITEMS")) {
109-
if (!cacheItemsAll()) {
108+
if (Config.CheckFilesType.valueOf(Config.get(Config.CF_MODE))
109+
== Config.CheckFilesType.DB_BATCH_ITEMS_PER_ITEM_ATTS_CONFIRM_INLINE) {
110+
if (!cacheItemsBatched(true)) {
110111
closeConnection();
111112
return false;
112113
}
113114
} else {
114-
if (!cacheItemsBatched()) {
115-
closeConnection();
116-
return false;
115+
// Cache Items
116+
if (Config.get(Config.CF_MODE).contains("ALL_ITEMS")) {
117+
if (!cacheItemsAll()) {
118+
closeConnection();
119+
return false;
120+
}
121+
} else {
122+
if (!cacheItemsBatched(false)) {
123+
closeConnection();
124+
return false;
125+
}
117126
}
118-
}
119127

120-
// Cache Attachments
121-
if (Config.get(Config.CF_MODE).endsWith("ALL_ATTACHMENTS")) {
122-
if (!cacheAttachmentsAll()) {
123-
closeConnection();
124-
return false;
125-
}
126-
} else {
127-
if (!cacheAttachmentsPerItem()) {
128-
closeConnection();
129-
return false;
128+
// Cache Attachments
129+
if (Config.get(Config.CF_MODE).endsWith("ALL_ATTACHMENTS")) {
130+
if (!cacheAttachmentsAll()) {
131+
closeConnection();
132+
return false;
133+
}
134+
} else {
135+
if (!cacheAttachmentsPerItem()) {
136+
closeConnection();
137+
return false;
138+
}
130139
}
131-
}
132140

133-
// Check attachments one item at a time
134-
int counter = 0;
135-
for (Integer itemId : attachmentsCache.keySet()) {
136-
counter++;
137-
if (!processItem(itemId, counter)) {
138-
closeConnection();
139-
return false;
141+
// Check attachments one item at a time
142+
int counter = 0;
143+
for (Integer itemId : attachmentsCache.keySet()) {
144+
counter++;
145+
if (!processItem(itemId, counter)) {
146+
closeConnection();
147+
return false;
148+
}
140149
}
141150
}
142151

@@ -443,33 +452,11 @@ private boolean cacheAttachmentsPerItem() {
443452
try {
444453
// For each item cached, query for the associated attachments.
445454
for (List<ResultsRow> itemRowSet : attachmentsCache.values()) {
446-
long batchStart = System.currentTimeMillis();
447-
448-
int itemId = itemRowSet.get(0).getItemId();
449-
PreparedStatement stmt =
450-
con.prepareStatement(
451-
"select a.type, a.url, a.value1, a.uuid, a.id "
452-
+ "from attachment a where a.item_id = ?");
453-
stmt.setInt(1, itemId);
454-
ResultSet rs = stmt.executeQuery();
455-
while (rs.next()) {
456-
counter++;
457-
ResultsRow attRow =
458-
inflateAttRow(
459-
rs.getString(1),
460-
rs.getString(2),
461-
rs.getString(3),
462-
rs.getString(4),
463-
rs.getInt(5),
464-
itemId);
465-
// Note - at this point, attRow will not be null.
466-
attachmentsCache.get(attRow.getItemId()).add(attRow);
467-
logger.info("Attachment ({}) gathered: {}", counter, attRow.toString());
455+
Pair<Boolean, Integer> result = cacheAttachmentsPerItem(itemRowSet.get(0).getItemId());
456+
if (!result.getFirst()) {
457+
return false;
468458
}
469-
rs.close();
470-
long batchDur = System.currentTimeMillis() - batchStart;
471-
ReportManager.getInstance().getStats().queryRan(batchDur);
472-
logger.debug("Cached a batch of attachments. Duration {} ms.", batchDur);
459+
counter += result.getSecond();
473460
}
474461
} catch (Exception e) {
475462
logger.error("Unable to cache attachments (query per item) - {}", e.getMessage(), e);
@@ -481,6 +468,46 @@ private boolean cacheAttachmentsPerItem() {
481468
return true;
482469
}
483470

471+
private Pair<Boolean, Integer> cacheAttachmentsPerItem(int itemId) {
472+
long start = System.currentTimeMillis();
473+
int counter = 0;
474+
try {
475+
long batchStart = System.currentTimeMillis();
476+
477+
PreparedStatement stmt =
478+
con.prepareStatement(
479+
"select a.type, a.url, a.value1, a.uuid, a.id "
480+
+ "from attachment a where a.item_id = ?");
481+
stmt.setInt(1, itemId);
482+
ResultSet rs = stmt.executeQuery();
483+
while (rs.next()) {
484+
counter++;
485+
ResultsRow attRow =
486+
inflateAttRow(
487+
rs.getString(1),
488+
rs.getString(2),
489+
rs.getString(3),
490+
rs.getString(4),
491+
rs.getInt(5),
492+
itemId);
493+
// Note - at this point, attRow will not be null.
494+
attachmentsCache.get(attRow.getItemId()).add(attRow);
495+
logger.info("Attachment ({}) gathered: {}", counter, attRow.toString());
496+
}
497+
rs.close();
498+
long batchDur = System.currentTimeMillis() - batchStart;
499+
ReportManager.getInstance().getStats().queryRan(batchDur);
500+
logger.debug("Cached a batch of attachments. Duration {} ms.", batchDur);
501+
} catch (Exception e) {
502+
logger.error("Unable to cache attachments (query per item) - {}", e.getMessage(), e);
503+
return Pair.pair(false, counter);
504+
}
505+
long dur = System.currentTimeMillis() - start;
506+
logger.info("Cached {} attachments (query per item). Duration {} ms.", counter, dur);
507+
ReportManager.getInstance().getStats().incNumGrandTotalAttachments(counter);
508+
return Pair.pair(true, counter);
509+
}
510+
484511
private boolean cacheAttachmentsAll() {
485512
long start = System.currentTimeMillis();
486513
int counter = 0;
@@ -704,7 +731,7 @@ private boolean cacheItemsAll() {
704731
*
705732
* @return
706733
*/
707-
private boolean cacheItemsBatched() {
734+
private boolean cacheItemsBatched(boolean confirmInline) {
708735
long start = System.currentTimeMillis();
709736
try {
710737
String sql =
@@ -755,6 +782,20 @@ private boolean cacheItemsBatched() {
755782
attachmentsCache.put(itemRow.getItemId(), seeder);
756783

757784
logger.info("Found the [{}]th item: {}", (attachmentsCache.size()), itemRow.toString());
785+
if (confirmInline) {
786+
Pair<Boolean, Integer> result = cacheAttachmentsPerItem(itemRow.getItemId());
787+
if (result.getFirst()) {
788+
if (!processItem(itemRow.getItemId(), attachmentsCache.size())) {
789+
rs.close();
790+
throw new Exception(
791+
"Unable to check attachments for item [" + itemRow.getItemId() + "]");
792+
}
793+
} else {
794+
rs.close();
795+
throw new Exception(
796+
"Unable to cache attachments for item [" + itemRow.getItemId() + "]");
797+
}
798+
}
758799
}
759800
rs.close();
760801
offsetCounter++;

‎src/test/java/org/apereo/openequella/tools/toolbox/CheckFilesDbTests.java

+55
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,61 @@ public void testAllInstitutions() {
260260
"# Of queries ran,30");
261261
}
262262

263+
@Test
264+
public void testAllInstitutionsConfirmInlineBatchedByDefault() {
265+
Config.reset();
266+
TestUtils.buildCheckFilesGeneralDbProps();
267+
Config.getInstance().setConfig(Config.CF_EMAIL_MODE, Config.CheckFilesEmailMode.NONE.name());
268+
Config.getInstance()
269+
.setConfig(
270+
Config.CF_MODE,
271+
Config.CheckFilesType.DB_BATCH_ITEMS_PER_ITEM_ATTS_CONFIRM_INLINE.name());
272+
Config.getInstance().checkConfigsCheckFiles();
273+
274+
assertTrue(CheckFilesDriver.setup(true));
275+
assertTrue(CheckFilesDriver.run());
276+
assertTrue(CheckFilesDriver.finalizeRun());
277+
278+
TestUtils.debugDumpList(ReportManager.getInstance().getAllStatsWriterList());
279+
TestUtils.debugDumpList(ReportManager.getInstance().getErrorStatsWriterList());
280+
confirmResultsAllInstitutions(
281+
ReportManager.getInstance().getAllStatsWriterList(),
282+
ReportManager.getInstance().getErrorStatsWriterList());
283+
assertEquals(
284+
"# Of queries ran,32",
285+
ReportManager.getInstance()
286+
.getAllStatsWriterList()
287+
.get(INST_ALL_NUM_OF_ALL_RESULTS - OFFSET_FOR_QUERY_STATEMENT));
288+
}
289+
290+
@Test
291+
public void testAllInstitutionsConfirmInlineBatchedBy1() {
292+
Config.reset();
293+
TestUtils.buildCheckFilesGeneralDbProps();
294+
Config.getInstance().setConfig(Config.CF_EMAIL_MODE, Config.CheckFilesEmailMode.NONE.name());
295+
Config.getInstance()
296+
.setConfig(
297+
Config.CF_MODE,
298+
Config.CheckFilesType.DB_BATCH_ITEMS_PER_ITEM_ATTS_CONFIRM_INLINE.name());
299+
Config.getInstance().setConfig(Config.CF_NUM_OF_ITEMS_PER_QUERY, "1");
300+
Config.getInstance().checkConfigsCheckFiles();
301+
302+
assertTrue(CheckFilesDriver.setup(true));
303+
assertTrue(CheckFilesDriver.run());
304+
assertTrue(CheckFilesDriver.finalizeRun());
305+
306+
TestUtils.debugDumpList(ReportManager.getInstance().getAllStatsWriterList());
307+
TestUtils.debugDumpList(ReportManager.getInstance().getErrorStatsWriterList());
308+
confirmResultsAllInstitutions(
309+
ReportManager.getInstance().getAllStatsWriterList(),
310+
ReportManager.getInstance().getErrorStatsWriterList());
311+
assertEquals(
312+
"# Of queries ran,56",
313+
ReportManager.getInstance()
314+
.getAllStatsWriterList()
315+
.get(INST_ALL_NUM_OF_ALL_RESULTS - OFFSET_FOR_QUERY_STATEMENT));
316+
}
317+
263318
@Test
264319
public void testAllInstitutionsBatchBy1() {
265320
Config.reset();

0 commit comments

Comments
 (0)