Neptune Analytics Support & Logging

.gitignore

@@ -12,4 +12,5 @@ test/TestCases/Case02/output/**
 test/TestCases/Case03/output/**
 test/TestCases/Case04/output/**
 test/TestCases/Case05/output/**
-test/TestCases/Case06/output/**
+test/TestCases/Case06/output/**
+*.iml

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aws/neptune-for-graphql",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "CLI utility to create and maintain a GraphQL API for Amazon Neptune",
   "keywords": [
     "Amazon Neptune",

samples/todo.schema.graphql

@@ -10,8 +10,4 @@ type Todo {
 
 type Comment {
     content: String
-}
-
-input Options {
-    limit: Int
 }

src/CDKPipelineApp.js

@@ -16,6 +16,7 @@ import { readFile, writeFile } from 'fs/promises';
 import fs from 'fs';
 import archiver from 'archiver';
 import ora from 'ora';
+import { loggerError, loggerLog } from "./logger.js";
 
 let NAME = '';
 let REGION = '';
@@ -32,7 +33,7 @@ let APPSYNC_ATTACH_QUERY = [];
 let APPSYNC_ATTACH_MUTATION = [];
 let SCHEMA_MODEL = null;
 let thisOutputFolderPath = './output';
-
+let msg = '';
 
 function yellow(text) {
     return '\x1b[33m' + text + '\x1b[0m';
@@ -68,7 +69,8 @@ async function createDeploymentFile(folderPath, zipFilePath) {
         archive.file('./output/output.resolver.graphql.js', { name: 'output.resolver.graphql.js' })
         await archive.finalize();
     } catch (err) {
-        console.error('Creating deployment zip file: ' + err); 
+        msg = 'Creating deployment zip file: ' + err;
+        loggerError(msg);
     }
 }
 
@@ -125,6 +127,8 @@ async function createAWSpipelineCDK (pipelineName, neptuneDBName, neptuneDBregio
         NEPTUNE_IAM_POLICY_RESOURCE = neptuneClusterInfo.iamPolicyResource;      
 
     } catch (error) {
+        msg = 'Error getting Neptune Cluster Info: ' + JSON.stringify(error);
+        loggerLog(msg);
         if (!quiet) spinner.fail("Error getting Neptune Cluster Info.");
         if (!isNeptuneIAMAuth) {
             spinner.clear();

src/NeptuneSchema.js

@@ -14,15 +14,17 @@ import axios from "axios";
 import { aws4Interceptor } from "aws4-axios";
 import { fromNodeProviderChain  } from "@aws-sdk/credential-providers";
 import { NeptunedataClient, ExecuteOpenCypherQueryCommand } from "@aws-sdk/client-neptunedata";
+import { loggerLog } from "./logger.js";
 
 let HOST = '';
 let PORT = 8182;
 let REGION = ''
 let SAMPLE = 5000;
-let VERBOSE = false; 
+let VERBOSE = false;
+let NEPTUNE_TYPE = 'neptune-db'; 
 let language = 'openCypher';
 let useSDK = false;
-
+let msg = '';
 
 async function getAWSCredentials() {
     const credentialProvider = fromNodeProviderChain();
@@ -31,7 +33,7 @@ async function getAWSCredentials() {
     const interceptor = aws4Interceptor({
         options: {
             region: REGION,
-            service: "neptune-db",
+            service: NEPTUNE_TYPE,
         },
         credentials: cred
     });
@@ -52,46 +54,68 @@ function yellow(text) {
 
 
 function consoleOut(text) {
-    if (VERBOSE) {
-        console.log(text);
-    }
+    loggerLog(text, VERBOSE);
 }
 
+function sanitize(text) {
+    // TODO implement sanitization logic
+    return text;
+}
 
-async function queryNeptune(q) {    
+/**
+ * Executes a neptune query
+ * @param query the query to execute
+ * @param params optional query params
+ * @returns {Promise<ExecuteOpenCypherQueryCommandOutput|any>}
+ */
+async function queryNeptune(query, params) {
     if (useSDK) {
-        const response = await queryNeptuneSDK(q);
-        return response; 
+        const response = await queryNeptuneSDK(query, params);
+        return response;
     } else {
-        try {        
-        const response = await axios.post(`https://${HOST}:${PORT}/${language}`, `query=${encodeURIComponent(q)}`);
-        return response.data;    
+        try {
+            let data = {
+                ...{query: query},
+                ...(params) && {parameters: params}
+            };
+            const response = await axios.post(`https://${HOST}:${PORT}/${language}`, data);
+        return response.data;
         } catch (error) {
-            console.error("Http query request failed: ", error.message);
-            consoleOut("Trying with the AWS SDK");            
-            const response = await queryNeptuneSDK(q);
-            useSDK = true;
-            return response;             
+            msg = `Http query request failed: ${error.message}`;
+            console.error(msg);
+            loggerLog(msg + ': ' + JSON.stringify(error));
+
+            if (NEPTUNE_TYPE == 'neptune-db') {
+                consoleOut("Trying with the AWS SDK");
+                const response = await queryNeptuneSDK(query, params);
+                useSDK = true;
+                return response;
+            }
+
+            throw new Error('AWS SDK for Neptune Analytics is not available, yet.');
         }
     } 
 }
 
 
-async function queryNeptuneSDK(q) {
+async function queryNeptuneSDK(query, params) {
     try {
         const config = {            
             endpoint: `https://${HOST}:${PORT}`
         };
         const client = new NeptunedataClient(config);
         const input = {
-            openCypherQuery: q            
+            ...{openCypherQuery: query},
+            ...(params) && {parameters: params}
         };
         const command = new ExecuteOpenCypherQueryCommand(input);
         const response = await client.send(command);        
         return response;
 
     } catch (error) {        
-        console.error("SDK query request failed: ", error.message);
+        msg = `SDK query request failed: ${error.message}`;
+        console.error(msg);
+        loggerLog(msg + ': ' + JSON.stringify(error));
         process.exit(1);
     }
 }
@@ -100,6 +124,7 @@ async function queryNeptuneSDK(q) {
 async function getNodesNames() {
     let query = `MATCH (a) RETURN labels(a), count(a)`;
     let response = await queryNeptune(query);    
+    loggerLog('Getting nodes names');
 
     try {
         response.results.forEach(result => {
@@ -108,7 +133,9 @@ async function getNodesNames() {
         });        
     }
     catch (e)  {
-        consoleOut("  No nodes found");
+        msg = "  No nodes found";
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
         return;
     }
 }
@@ -117,6 +144,7 @@ async function getNodesNames() {
 async function getEdgesNames() {
     let query = `MATCH ()-[e]->() RETURN type(e), count(e)`;
     let response = await queryNeptune(query);
+    loggerLog('Getting edges names');
 
     try {
         response.results.forEach(result => {
@@ -125,35 +153,31 @@ async function getEdgesNames() {
         });
     }
     catch (e)  {
-        consoleOut("  No edges found");
+        msg = "  No edges found";
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
         return;
     }
 
 }
 
 
-async function checkEdgeDirection(direction) {
-    let query = `MATCH (from:${direction.from})-[r:${direction.edge.label}]->(to:${direction.to}) RETURN r as edge LIMIT 1`;
+async function findFromAndToLabels(edgeStructure) {
+    let query = `MATCH (from)-[r:${sanitize(edgeStructure.label)}]->(to) RETURN DISTINCT labels(from) as fromLabel, labels(to) as toLabel`;
     let response = await queryNeptune(query);
-    let result = response.results[0];
-    if (result !== undefined) {                    
-        direction.edge.directions.push({from:direction.from, to:direction.to});
-        consoleOut('  Found edge: ' + yellow(direction.edge.label) + '  direction: ' + yellow(direction.from) + ' -> ' + yellow(direction.to));
+    for (let result of response.results) {
+        for (let fromLabel of result.fromLabel) {
+            for (let toLabel of result.toLabel) {
+                edgeStructure.directions.push({from:fromLabel, to:toLabel});
+                consoleOut('  Found edge: ' + yellow(edgeStructure.label) + '  direction: ' + yellow(fromLabel) + ' -> ' + yellow(toLabel));
+            }
+        }
     }
 }
 
 
 async function getEdgesDirections() {
-    let possibleDirections = [];
-    for (const edge of schema.edgeStructures) {        
-        for (const fromNode of schema.nodeStructures) {
-            for (const toNode of schema.nodeStructures) {
-                possibleDirections.push({edge:edge, from:fromNode.label, to:toNode.label});            
-            }
-        }
-    }
-
-    await Promise.all(possibleDirections.map(checkEdgeDirection))
+    await Promise.all(schema.edgeStructures.map(findFromAndToLabels))
 }
 
 
@@ -196,9 +220,10 @@ function addUpdateEdgeProperty(edgeName, name, value) {
 
 
 async function getEdgeProperties(edge) {
-    let query = `MATCH ()-[n:${edge.label}]->() RETURN properties(n) as properties LIMIT ${SAMPLE}`;        
+    let query = `MATCH ()-[n:${sanitize(edge.label)}]->() RETURN properties(n) as properties LIMIT $sample`;
+    let parameters = `{"sample": ${SAMPLE}}`;
     try {
-        let response = await queryNeptune(query);            
+        let response = await queryNeptune(query, parameters);            
         let result = response.results;
         result.forEach(e => {                                
             Object.keys(e.properties).forEach(key => {
@@ -207,7 +232,9 @@ async function getEdgeProperties(edge) {
         });            
     }
     catch (e)  {
-        consoleOut("  No properties found for edge: " + edge.label);
+        msg = "  No properties found for edge: " + edge.label;
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
     }    
 }
 
@@ -220,9 +247,10 @@ async function getEdgesProperties() {
 
 
 async function getNodeProperties(node) {
-    let query = `MATCH (n:${node.label}) RETURN properties(n) as properties LIMIT ${SAMPLE}`;        
+    let query = `MATCH (n:${sanitize(node.label)}) RETURN properties(n) as properties LIMIT $sample`;
+    let parameters = `{"sample": ${SAMPLE}}`;
     try {
-        let response = await queryNeptune(query);            
+        let response = await queryNeptune(query, parameters);
         let result = response.results;
         result.forEach(e => {                                
             Object.keys(e.properties).forEach(key => {
@@ -231,7 +259,9 @@ async function getNodeProperties(node) {
         });            
     }
     catch (e)  {
-        consoleOut("  No properties found for node: " + node.label);
+        msg = "  No properties found for node: " + node.label;
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(e));
     }    
 }
 
@@ -244,10 +274,10 @@ async function getNodesProperties() {
 
 
 async function checkEdgeDirectionCardinality(d) {
-    let queryFrom = `MATCH (from:${d.from})-[r:${d.edge.label}]->(to:${d.to}) WITH to, count(from) as rels WHERE rels > 1 RETURN rels LIMIT 1`;     
+    let queryFrom = `MATCH (from:${sanitize(d.from)})-[r:${sanitize(d.edge.label)}]->(to:${sanitize(d.to)}) WITH to, count(from) as rels WHERE rels > 1 RETURN rels LIMIT 1`;
     let responseFrom = await queryNeptune(queryFrom);
     let resultFrom = responseFrom.results[0];
-    let queryTo = `MATCH (from:${d.from})-[r:${d.edge.label}]->(to:${d.to}) WITH from, count(to) as rels WHERE rels > 1 RETURN rels LIMIT 1`;            
+    let queryTo = `MATCH (from:${sanitize(d.from)})-[r:${sanitize(d.edge.label)}]->(to:${sanitize(d.to)}) WITH from, count(to) as rels WHERE rels > 1 RETURN rels LIMIT 1`;
     let responseTo = await queryNeptune(queryTo);
     let resultTo = responseTo.results[0];
     let c = '';
@@ -283,11 +313,12 @@ async function getEdgesDirectionsCardinality() {
 }
 
 
-function setGetNeptuneSchemaParameters(host, port, region, verbose = false) {
+function setGetNeptuneSchemaParameters(host, port, region, verbose = false, neptuneType) {
     HOST = host;
     PORT = port;
     REGION = region;
     VERBOSE = verbose;
+    NEPTUNE_TYPE = neptuneType;
 }
 
 
@@ -307,6 +338,7 @@ async function getSchemaViaSummaryAPI() {
         return true;
 
     } catch (error) {
+        loggerLog(`Getting the schema via Neptune Summary API failed: ${JSON.stringify(error)}`);
         return false;
     }    
 }
@@ -318,8 +350,10 @@ async function getNeptuneSchema(quiet) {
 
     try {
         await getAWSCredentials();
-    } catch (error) {        
-        consoleOut("There are no AWS credetials configured. \nGetting the schema from an Amazon Neptune database with IAM authentication works only with AWS credentials.");
+    } catch (error) {
+        msg = "There are no AWS credentials configured. \nGetting the schema from an Amazon Neptune database with IAM authentication works only with AWS credentials.";
+        consoleOut(msg);
+        loggerLog(msg + ': ' + JSON.stringify(error));
     }
 
     if (await getSchemaViaSummaryAPI()) {

src/graphdb.js

@@ -69,12 +69,16 @@ function graphDBInferenceSchema (graphbSchema, addMutations) {
         }
 
         r += '\t_id: ID! @id\n';
-
+
+        let properties = [];
         node.properties.forEach(property => {
+            properties.push(property.name);
+
             if (property.name == 'id')                
                 r+= `\tid: ID\n`;
             else
                 r+= `\t${property.name}: ${property.type}\n`;
+
         });
 
         let edgeTypes = [];
@@ -127,11 +131,15 @@ function graphDBInferenceSchema (graphbSchema, addMutations) {
         });
 
         // Add edge types        
-        edgeTypes.forEach(edgeType => {
+        edgeTypes.forEach(edgeType => {            
+            let collision = '';
+            if (properties.includes(edgeType))
+                collision = '_';
+
             if (changeCase) {
-                r += `\t${edgeType}:${toPascalCase(edgeType)}`
+                r += `\t${collision + edgeType}:${toPascalCase(edgeType)}`
             } else {
-                r += `\t${edgeType}:${edgeType}`
+                r += `\t${collision + edgeType}:${edgeType}`
             }
         });