Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexzerntev
alexzerntev committed Mar 17, 2024
1 parent bd29f6c commit 6ba4685
Show file tree
Hide file tree
Showing 14 changed files with 152 additions and 161 deletions.
2 changes: 1 addition & 1 deletion snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/BenchmarkTests.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ trait BenchmarkTests extends CompilerTestContext {
}

test("Range Join File with db test tpch10") { _ =>
assume(true, "This test is disabled by default")
assume(false, "This test is disabled by default")

val prog = """let
| lineitemsType = type collection(record(l_orderkey: int, l_payedammount: double)),
Expand Down
30 changes: 15 additions & 15 deletions ...ient/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerServiceTestContext.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,21 @@ trait Rql2TruffleCompilerServiceTestContext extends Rql2CompilerServiceTestConte
val options = new java.util.HashMap[String, String]()
options.put("rql.settings", settings.renderAsString)
// //diagnostics
options.put("engine.CompilationStatistics", "true")
// options.put("engine.CompilationStatisticDetails", "true")
// options.put("compiler.InstrumentBoundaries", "true")
options.put("engine.CompilationFailureAction", "Diagnose")
options.put("engine.TracePerformanceWarnings", "all")
options.put("engine.TraceCompilation", "true")
options.put("engine.TraceSplitting", "true")
options.put("engine.TraceDeoptimizeFrame", "true")
options.put("engine.TraceTransferToInterpreter", "true")
options.put("engine.TraceCompilationPolymorphism", "true")
options.put("engine.TraceSplittingSummary", "true")
// options.put("engine.TraceCompilationDetails", "true")
// options.put("engine.CompileImmediately", "true")
options.put("engine.BackgroundCompilation", "false")
options.put("engine.SpecializationStatistics", "false")
// options.put("engine.CompilationStatistics", "true")
// // options.put("engine.CompilationStatisticDetails", "true")
// // options.put("compiler.InstrumentBoundaries", "true")
// options.put("engine.CompilationFailureAction", "Diagnose")
// options.put("engine.TracePerformanceWarnings", "all")
// options.put("engine.TraceCompilation", "true")
// options.put("engine.TraceSplitting", "true")
// options.put("engine.TraceDeoptimizeFrame", "true")
// options.put("engine.TraceTransferToInterpreter", "true")
// options.put("engine.TraceCompilationPolymorphism", "true")
// options.put("engine.TraceSplittingSummary", "true")
// // options.put("engine.TraceCompilationDetails", "true")
//// options.put("engine.CompileImmediately", "true")
// options.put("engine.BackgroundCompilation", "false")
// options.put("engine.SpecializationStatistics", "false")

// // optimizations
// options.put("compiler.InlineAcrossTruffleBoundary", "true")
Expand Down
8 changes: 1 addition & 7 deletions ...ffle/src/main/java/raw/runtime/truffle/ast/expressions/iterable/list/ListGroupByNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@
import raw.runtime.truffle.ast.osr.conditions.OSRHasNextConditionNode;
import raw.runtime.truffle.runtime.generator.collection.GeneratorNodes;
import raw.runtime.truffle.runtime.generator.collection.GeneratorNodesFactory;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.OffHeapNodes;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.OffHeapNodesFactory;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.group_by.OffHeapGroupByKey;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.record_shaper.RecordShaper;
import raw.runtime.truffle.runtime.iterable.IterableNodes;
Expand Down Expand Up @@ -62,10 +60,6 @@ public class ListGroupByNode extends ExpressionNode {
private GeneratorNodes.GeneratorCloseNode generatorCloseNode =
GeneratorNodesFactory.GeneratorCloseNodeGen.create();

@Child
OffHeapNodes.OffHeapGeneratorNode generatorNode =
OffHeapNodesFactory.OffHeapGeneratorNodeGen.create();

private final Rql2TypeWithProperties rowType;
private final Rql2TypeWithProperties keyType;

Expand Down Expand Up @@ -148,7 +142,7 @@ public Object executeGeneric(VirtualFrame frame) {
} finally {
generatorCloseNode.execute(this, generator);
}
Object mapGenerator = generatorNode.execute(this, map);
Object mapGenerator = map.getGenerator();
try {
generatorInitNode.execute(this, mapGenerator);
frame.setObject(generatorSlot, mapGenerator);
Expand Down
12 changes: 12 additions & 0 deletions ...-truffle/src/main/java/raw/runtime/truffle/ast/osr/bodies/OSROffHeapDistinctBodyNode.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
/*
* Copyright 2023 RAW Labs S.A.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/

package raw.runtime.truffle.ast.osr.bodies;

import com.esotericsoftware.kryo.io.Output;
Expand Down
12 changes: 12 additions & 0 deletions ...ruffle/src/main/java/raw/runtime/truffle/ast/osr/bodies/OSROffHeapGroupByKeyBodyNode.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
/*
* Copyright 2023 RAW Labs S.A.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/

package raw.runtime.truffle.ast.osr.bodies;

import static raw.runtime.truffle.runtime.generator.collection.off_heap_generator.StaticOffHeap.kryoWriteInt;
Expand Down
12 changes: 12 additions & 0 deletions ...uffle/src/main/java/raw/runtime/truffle/ast/osr/bodies/OSROffHeapGroupByKeysBodyNode.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
/*
* Copyright 2023 RAW Labs S.A.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/

package raw.runtime.truffle.ast.osr.bodies;

import static raw.runtime.truffle.runtime.generator.collection.off_heap_generator.StaticOffHeap.kryoWriteInt;
Expand Down
12 changes: 12 additions & 0 deletions .../src/main/java/raw/runtime/truffle/ast/osr/conditions/OSRTreeMapHasNextConditionNode.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
/*
* Copyright 2023 RAW Labs S.A.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/

package raw.runtime.truffle.ast.osr.conditions;

import com.oracle.truffle.api.frame.VirtualFrame;
Expand Down
9 changes: 3 additions & 6 deletions ...ruffle/runtime/generator/collection/abstract_generator/compute_next/ComputeNextNodes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
import raw.runtime.truffle.runtime.generator.collection.GeneratorNodes;
import raw.runtime.truffle.runtime.generator.collection.abstract_generator.compute_next.operations.*;
import raw.runtime.truffle.runtime.generator.collection.abstract_generator.compute_next.sources.*;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.OffHeapNodes;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.group_by.OffHeapGroupByKey;
import raw.runtime.truffle.runtime.iterable.IterableNodes;
import raw.runtime.truffle.runtime.iterable.sources.EmptyCollection;
Expand Down Expand Up @@ -713,9 +712,7 @@ static void init(
GeneratorNodes.GeneratorInitNode initRightNode,
@Cached @Cached.Shared("close1") GeneratorNodes.GeneratorCloseNode closeLeftNode,
@SuppressWarnings("truffle-sharing") @Cached
GeneratorNodes.GeneratorCloseNode closeRightNode,
@Cached OffHeapNodes.OffHeapGeneratorNode offHeapGeneratorLeft,
@Cached OffHeapNodes.OffHeapGeneratorNode offHeapGeneratorRight) {
GeneratorNodes.GeneratorCloseNode closeRightNode) {
Frame frame = computeNext.getFrame();
// left side (get a generator, then fill a map, set leftMapGenerator to the map generator)
OffHeapGroupByKey leftMap =
Expand All @@ -739,7 +736,7 @@ static void init(
} finally {
closeLeftNode.execute(thisNode, leftGenerator);
}
computeNext.setLeftMapGenerator(offHeapGeneratorLeft.execute(thisNode, leftMap));
computeNext.setLeftMapGenerator(leftMap.getGenerator());
initLeftNode.execute(thisNode, computeNext.getLeftMapGenerator());

// same with right side
Expand All @@ -763,7 +760,7 @@ static void init(
} finally {
closeRightNode.execute(thisNode, rightGenerator);
}
computeNext.setRightMapGenerator(offHeapGeneratorRight.execute(thisNode, rightMap));
computeNext.setRightMapGenerator(rightMap.getGenerator());
initRightNode.execute(thisNode, computeNext.getRightMapGenerator());
}

Expand Down
12 changes: 12 additions & 0 deletions ...va/raw/runtime/truffle/runtime/generator/collection/off_heap_generator/StaticOffHeap.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
/*
* Copyright 2023 RAW Labs S.A.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/

package raw.runtime.truffle.runtime.generator.collection.off_heap_generator;

import com.esotericsoftware.kryo.io.Output;
Expand Down
164 changes: 42 additions & 122 deletions ...untime/truffle/runtime/generator/collection/off_heap_generator/off_heap/OffHeapNodes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,9 @@
import raw.runtime.truffle.runtime.data_structures.treemap.TreeMapIterator;
import raw.runtime.truffle.runtime.data_structures.treemap.TreeMapNodes;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.StaticOffHeap;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.distinct.DistinctMemoryGenerator;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.distinct.DistinctSpilledFilesGenerator;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.distinct.OffHeapDistinct;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.group_by.GroupByMemoryGenerator;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.group_by.GroupBySpilledFilesGenerator;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.group_by.OffHeapGroupByKey;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.order_by.OffHeapGroupByKeys;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.order_by.OrderByMemoryGenerator;
import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.order_by.OrderBySpilledFilesGenerator;

public class OffHeapNodes {
@NodeInfo(shortName = "OffHeap.Put")
Expand Down Expand Up @@ -182,120 +176,46 @@ static void put(
}
}
}

@NodeInfo(shortName = "OffHeap.Generator")
@GenerateUncached
@GenerateInline
@ImportStatic(StaticOffHeap.class)
public abstract static class OffHeapGeneratorNode extends Node {

public abstract Object execute(Node node, Object offHeap);

@Specialization(guards = "offHeapGroupByKey.getSpilledBuffers().isEmpty()")
static GroupByMemoryGenerator generator(Node node, OffHeapGroupByKey offHeapGroupByKey) {
return new GroupByMemoryGenerator(offHeapGroupByKey);
}

@Specialization(guards = "!offHeapGroupByKey.getSpilledBuffers().isEmpty()")
static GroupBySpilledFilesGenerator generator(
Node node,
OffHeapGroupByKey offHeapGroupByKey,
@Bind("$node") Node thisNode,
@Cached(
value = "getOffHeapGroupByKeyLoopNode(offHeapGroupByKey)",
allowUncached = true,
inline = false,
neverDefault = true)
LoopNode loopNode) {
// flush
Output kryoOutput =
new UnsafeOutput(
getGroupByKeyNewDiskBuffer(offHeapGroupByKey, thisNode),
offHeapGroupByKey.getKryoOutputBufferSize());
TreeMapIterator iterator = offHeapGroupByKey.getMemMap().iterator();

MaterializedFrame frame = offHeapGroupByKey.getFrame();
frame.setAuxiliarySlot(offHeapGroupByKey.getKryoOutputSlot(), kryoOutput);
frame.setAuxiliarySlot(offHeapGroupByKey.getIteratorSlot(), iterator);
frame.setAuxiliarySlot(offHeapGroupByKey.getOffHeapGroupByKeySlot(), offHeapGroupByKey);
loopNode.execute(frame);

kryoOutputClose(kryoOutput);
// reset both the memory map and memory footprint.

offHeapGroupByKey.getMemMap().clear();
offHeapGroupByKey.setSize(0);
return new GroupBySpilledFilesGenerator(offHeapGroupByKey);
}

@Specialization(guards = "offHeapGroupByKeys.getSpilledBuffers().isEmpty()")
static OrderByMemoryGenerator generator(Node node, OffHeapGroupByKeys offHeapGroupByKeys) {
return new OrderByMemoryGenerator(offHeapGroupByKeys);
}

@Specialization(guards = "!offHeapGroupByKeys.getSpilledBuffers().isEmpty()")
static OrderBySpilledFilesGenerator generator(
Node node,
OffHeapGroupByKeys offHeapGroupByKeys,
@Bind("$node") Node thisNode,
@Cached(
value = "getOffHeapGroupByKeysLoopNode(offHeapGroupByKeys)",
allowUncached = true,
inline = false,
neverDefault = true)
LoopNode loopNode) {

Output kryoOutput =
new UnsafeOutput(
groupByKeysNextFile(offHeapGroupByKeys, thisNode),
offHeapGroupByKeys.getKryoOutputBufferSize());
TreeMapIterator iterator = offHeapGroupByKeys.getMemMap().iterator();

MaterializedFrame frame = offHeapGroupByKeys.getFrame();
frame.setAuxiliarySlot(offHeapGroupByKeys.getKryoOutputSlot(), kryoOutput);
frame.setAuxiliarySlot(offHeapGroupByKeys.getIteratorSlot(), iterator);
frame.setAuxiliarySlot(offHeapGroupByKeys.getOffHeapGroupByKeysSlot(), offHeapGroupByKeys);
loopNode.execute(frame);

kryoOutputClose(kryoOutput);
// reset the memory map and footprint
offHeapGroupByKeys.getMemMap().clear();
offHeapGroupByKeys.setSize(0);

return new OrderBySpilledFilesGenerator(offHeapGroupByKeys);
}

@Specialization(guards = "offHeapDistinct.getSpilledBuffers().isEmpty()")
static DistinctMemoryGenerator generator(Node node, OffHeapDistinct offHeapDistinct) {
return new DistinctMemoryGenerator(offHeapDistinct);
}

@Specialization(guards = "!offHeapDistinct.getSpilledBuffers().isEmpty()")
static DistinctSpilledFilesGenerator generator(
Node node,
OffHeapDistinct offHeapDistinct,
@Cached(
value = "getOffHeapDistinctLoopNode(offHeapDistinct)",
allowUncached = true,
inline = false,
neverDefault = true)
LoopNode loopNode) {
Output kryoOutput =
new UnsafeOutput(
distinctNextFile(offHeapDistinct, node), offHeapDistinct.getKryoInputBufferSize());
TreeMapIterator iterator = offHeapDistinct.getIndex().iterator();

MaterializedFrame frame = offHeapDistinct.getFrame();
frame.setAuxiliarySlot(offHeapDistinct.getKryoOutputSlot(), kryoOutput);
frame.setAuxiliarySlot(offHeapDistinct.getIteratorSlot(), iterator);
frame.setAuxiliarySlot(offHeapDistinct.getOffHeapDistinctSlot(), offHeapDistinct);
loopNode.execute(frame);

kryoOutputClose(kryoOutput);
// reset the memory map and footprint
offHeapDistinct.getIndex().clear();
offHeapDistinct.setBinarySize(0);
return new DistinctSpilledFilesGenerator(offHeapDistinct);
}
}
//
// @NodeInfo(shortName = "OffHeap.Generator")
// @GenerateUncached
// @GenerateInline
// @ImportStatic(StaticOffHeap.class)
// public abstract static class OffHeapGeneratorNode extends Node {
//
// public abstract Object execute(Node node, Object offHeap);
//
// @Specialization(guards = "offHeapGroupByKey.getSpilledBuffers().isEmpty()")
// static GroupByMemoryGenerator generator(Node node, OffHeapGroupByKey offHeapGroupByKey) {
// return new GroupByMemoryGenerator(offHeapGroupByKey);
// }
//
// @Specialization(guards = "!offHeapGroupByKey.getSpilledBuffers().isEmpty()")
// static GroupBySpilledFilesGenerator generatorSpilled(
// Node node, OffHeapGroupByKey offHeapGroupByKey) {
// return new GroupBySpilledFilesGenerator(offHeapGroupByKey);
// }
//
// @Specialization(guards = "offHeapGroupByKeys.getSpilledBuffers().isEmpty()")
// static OrderByMemoryGenerator generator(Node node, OffHeapGroupByKeys offHeapGroupByKeys) {
// return new OrderByMemoryGenerator(offHeapGroupByKeys);
// }
//
// @Specialization(guards = "!offHeapGroupByKeys.getSpilledBuffers().isEmpty()")
// static OrderBySpilledFilesGenerator generatorSpilled(
// Node node, OffHeapGroupByKeys offHeapGroupByKeys) {
// return new OrderBySpilledFilesGenerator(offHeapGroupByKeys);
// }
//
// @Specialization(guards = "offHeapDistinct.getSpilledBuffers().isEmpty()")
// static DistinctMemoryGenerator generator(Node node, OffHeapDistinct offHeapDistinct) {
// return new DistinctMemoryGenerator(offHeapDistinct);
// }
//
// @Specialization(guards = "!offHeapDistinct.getSpilledBuffers().isEmpty()")
// static DistinctSpilledFilesGenerator generatorSpilled(
// Node node, OffHeapDistinct offHeapDistinct) {
// return new DistinctSpilledFilesGenerator(offHeapDistinct);
// }
// }
}
8 changes: 8 additions & 0 deletions ...le/runtime/generator/collection/off_heap_generator/off_heap/distinct/OffHeapDistinct.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,14 @@ public SourceContext getContext() {
return context;
}

public Object getGenerator() {
if (this.getSpilledBuffers().isEmpty()) {
return new DistinctMemoryGenerator(this);
} else {
return new DistinctSpilledFilesGenerator(this);
}
}

public MaterializedFrame getFrame() {
return frame;
}
Expand Down
Loading

0 comments on commit 6ba4685

Please sign in to comment.