Merge pull request #79 from leben-in-deutschland/feature/inform-user-…

…about-new-questions

Add ID to stop api calls for translation

src/app/web/data/prüfstellen.json

@@ -2543,6 +2543,15 @@
         "telefon": "040-3199160-16",
         "email": "olga.debus@kom-bildung.de"
       },
+      {
+        "regierungsbezirk": "Kreisfreie Stadt",
+        "plz": 22767,
+        "ort": "Hamburg",
+        "einrichtung": "Akademikerbund Hamburg e.V.",
+        "straße": "Große Bergstrße 6a",
+        "telefon": "040-63737260",
+        "email": "altona@abh-kurse.de"
+      },
       {
         "regierungsbezirk": "Kreisfreie Stadt",
         "plz": 22767,
@@ -2919,8 +2928,8 @@
         "ort": "Wolfenbüttel",
         "einrichtung": "KVHS Wolfenbüttel",
         "straße": "Harzstr.2-5",
-        "telefon": "05331/84 157 oder ab Jan.09 05331/84 158",
-        "email": "d.eicke@lkwf.de"
+        "telefon": "05331-84-898 oder84-153",
+        "email": "biz-infopoint@lk-wf.de"
       },
       {
         "regierungsbezirk": "Verden",

src/app/web/package.json

@@ -63,4 +63,4 @@
   "keywords": [],
   "author": "",
   "license": "ISC"
-}
+}

src/app/web/types/question.ts

@@ -8,6 +8,7 @@ export const categories = ["Rights & Freedoms",
 
 export interface Question {
     num: string;
+    id: string;
     question: string;
     a: string;
     b: string;

src/scrap/data/prüfstellen.json

@@ -2543,6 +2543,15 @@
         "telefon": "040-3199160-16",
         "email": "olga.debus@kom-bildung.de"
       },
+      {
+        "regierungsbezirk": "Kreisfreie Stadt",
+        "plz": 22767,
+        "ort": "Hamburg",
+        "einrichtung": "Akademikerbund Hamburg e.V.",
+        "straße": "Große Bergstrße 6a",
+        "telefon": "040-63737260",
+        "email": "altona@abh-kurse.de"
+      },
       {
         "regierungsbezirk": "Kreisfreie Stadt",
         "plz": 22767,
@@ -2919,8 +2928,8 @@
         "ort": "Wolfenbüttel",
         "einrichtung": "KVHS Wolfenbüttel",
         "straße": "Harzstr.2-5",
-        "telefon": "05331/84 157 oder ab Jan.09 05331/84 158",
-        "email": "d.eicke@lkwf.de"
+        "telefon": "05331-84-898 oder84-153",
+        "email": "biz-infopoint@lk-wf.de"
       },
       {
         "regierungsbezirk": "Verden",

src/scrap/index.ts

@@ -4,6 +4,8 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as XLSX from 'xlsx';
 import { Prüfstellen } from './types/prüfstellen';
+import { createHash } from 'crypto';
+import existingQuestionJson from './data/question.json';
 
 const BASE_URL = 'https://www.einbuergerungstest-online.de';
 const BASE_URL_BAMF = "https://www.bamf.de"
@@ -48,7 +50,8 @@ const scrap = async (url: string, state: string) => {
             image: '-',
             translation: {},
             category: null,
-            context: ''
+            context: '',
+            id: ''
         };
         question.question = pageData(element).find("strong.font-semibold").text().trim();
         if (pageData(element).find("img").length > 0) {
@@ -153,9 +156,15 @@ const translate = async (question: Question) => {
     }
 };
 
+const generateId = (question: Question) => {
+    const text = question.question + question.a + question.b + question.c + question.d;
+    const crypt = createHash('sha256').update(text).digest('hex')
+    return crypt;
+};
+
 async function scrapeData() {
     try {
-
+        const oldQuestion = JSON.parse(JSON.stringify(existingQuestionJson)) as Question[]
         let questions = await scrapAll();
         let stateQuestion = await scrapStates();
         const allQuestion = [...questions, ...stateQuestion];
@@ -164,9 +173,17 @@ async function scrapeData() {
             if (!allQuestion[i].question) {
                 continue;
             }
-            allQuestion[i].category = await findCategory(allQuestion[i]);
-            allQuestion[i].context = await getContext(allQuestion[i]);
-            allQuestion[i] = await translate(allQuestion[i]);
+            allQuestion[i].id = generateId(allQuestion[i]);
+            const existing = oldQuestion.findIndex((q) => q && q.id === allQuestion[i].id);
+            if (existing !== -1) {
+                allQuestion[i].translation = oldQuestion[existing].translation;
+                allQuestion[i].category = oldQuestion[existing].category;
+                allQuestion[i].context = oldQuestion[existing].context;
+            } else {
+                allQuestion[i].category = await findCategory(allQuestion[i]);
+                allQuestion[i].context = await getContext(allQuestion[i]);
+                allQuestion[i] = await translate(allQuestion[i]);
+            }
         }
         const dir = './data';
         const filePath = path.join(dir, 'question.json');

src/scrap/tsconfig.json

@@ -1,6 +1,8 @@
 {
     "compilerOptions": {
         "target": "es5",
-        "sourceMap": true
+        "sourceMap": true,
+        "resolveJsonModule": true,
+        "esModuleInterop": true,
     }
 }

src/scrap/types/question.ts

@@ -1,5 +1,6 @@
 export interface Question {
     num: string;
+    id: string;
     question: string;
     a: string;
     b: string;