Skip to content

Commit

Permalink
Minor changes to class and variable names.
Browse files Browse the repository at this point in the history
  • Loading branch information
contactsunny committed Apr 2, 2020
1 parent ae0b36c commit 5eea4b4
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 16 deletions.
2 changes: 0 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
</build>

<dependencies>

<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
Expand All @@ -47,6 +46,5 @@
<artifactId>hadoop-core</artifactId>
<version>1.2.1</version>
</dependency>

</dependencies>
</project>
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.contactsunny.poc.parquet_file_writer_poc;

import com.contactsunny.poc.parquet_file_writer_poc.parquet.CsvParquetWriter;
import com.contactsunny.poc.parquet_file_writer_poc.parquet.CustomParquetWriter;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.schema.MessageType;
Expand All @@ -24,8 +24,8 @@ public class App implements CommandLineRunner {
@Value("${schema.filePath}")
private String schemaFilePath;

@Value("${output.filePath}")
private String outputPath;
@Value("${output.directoryPath}")
private String outputDirectoryPath;

private static final Logger logger = LoggerFactory.getLogger(App.class);

Expand All @@ -38,7 +38,7 @@ public void run(String... args) throws Exception {

List<List<String>> columns = getDataForFile();
MessageType schema = getSchemaForParquetFile();
CsvParquetWriter writer = getParquetWriter(schema);
CustomParquetWriter writer = getParquetWriter(schema);

for (List<String> column : columns) {
logger.info("Writing line: " + column.toArray());
Expand All @@ -49,11 +49,11 @@ public void run(String... args) throws Exception {
writer.close();
}

private CsvParquetWriter getParquetWriter(MessageType schema) throws IOException {
String outputFilePath = outputPath + "/" + System.currentTimeMillis() + ".parquet";
private CustomParquetWriter getParquetWriter(MessageType schema) throws IOException {
String outputFilePath = outputDirectoryPath + "/" + System.currentTimeMillis() + ".parquet";
File outputParquetFile = new File(outputFilePath);
Path path = new Path(outputParquetFile.toURI().toString());
return new CsvParquetWriter(
return new CustomParquetWriter(
path, schema, false, CompressionCodecName.SNAPPY
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,14 @@
import java.io.IOException;
import java.util.List;

public class CsvParquetWriter extends ParquetWriter<List<String>> {
public class CustomParquetWriter extends ParquetWriter<List<String>> {

public CsvParquetWriter(
public CustomParquetWriter(
Path file,
MessageType schema,
boolean enableDictionary,
CompressionCodecName codecName
) throws IOException {

super(file, new CsvWriteSupport(schema), codecName, DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE, enableDictionary, false);
super(file, new CustomWriteSupport(schema), codecName, DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE, enableDictionary, false);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@
import java.util.HashMap;
import java.util.List;

public class CsvWriteSupport extends WriteSupport<List<String>> {
public class CustomWriteSupport extends WriteSupport<List<String>> {
MessageType schema;
RecordConsumer recordConsumer;
List<ColumnDescriptor> cols;

// TODO: support specifying encodings and compression
CsvWriteSupport(MessageType schema) {
CustomWriteSupport(MessageType schema) {
this.schema = schema;
this.cols = schema.getColumns();
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/application.properties
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
schema.filePath=
output.filePath=
output.directoryPath=

0 comments on commit 5eea4b4

Please sign in to comment.