Merge branch 'main' into test-updates

Author: Malar-Natarajan

concourse/update-content/script.bash

@@ -11,13 +11,19 @@ fi
 # this is here so the creds don't get pasted to the output
 set -e; if [ -n "$DEBUG" ]; then set -x; fi
 
-approved_books_default_branch=$(curl -s https://api.github.com/repos/openstax/content-manager-approved-books | jq -r .default_branch)
 rex_default_branch=$(curl -s https://api.github.com/repos/openstax/rex-web | jq -r .default_branch)
 
-data=$(curl -sL "https://github.com/openstax/content-manager-approved-books/raw/$approved_books_default_branch/approved-book-list.json")
-
-# script will return a JSON object with book ids and new versions only for books that doesn't mach current config
-book_ids_and_versions=$(node script/entry.js transform-approved-books-data --data "$data")
+# consumer: Limit entries to those where consumer == <consumer>
+# code_version: Limit entries to those where code_version <= <code_version>
+archive_version="$(jq -r '.REACT_APP_ARCHIVE' "src/config.archive-url.json")"
+search="consumer=REX&code_version=$archive_version"
+abl_url="https://corgi.ce.openstax.org/api/abl/?$search"
+
+book_ids_and_versions="$(
+  bash script/transform-approved-books-data.bash \
+    <(curl -sL --fail --show-error "$abl_url") \
+    "src/config.books.json"
+)"
 book_entries=$(echo "$book_ids_and_versions" | jq -c 'to_entries | .[]')
 
 git remote set-branches origin 'update-content-*'

data/redirects/7193a423-3ad7-4ac6-98dd-90c70e4724b7.json

@@ -0,0 +1,7 @@
+[
+  {
+    "bookId": "7193a423-3ad7-4ac6-98dd-90c70e4724b7",
+    "pageId": "5ce4d68c-f5a2-4e6f-92ed-87da9ba26264",
+    "pathname": "/books/contemporary-mathematics/pages/5-6-quadratic-equations-with-two-variables-with-applications"
+  }
+]

data/redirects/d50f6e32-0fda-46ef-a362-9bd36ca7c97d.json

@@ -0,0 +1,7 @@
+[
+  {
+    "bookId": "d50f6e32-0fda-46ef-a362-9bd36ca7c97d",
+    "pageId": "a72e87ba-5cfb-4899-b11e-60d7c25ae36a",
+    "pathname": "/books/university-physics-volume-1/pages/4-4-uniform-circular-motion"
+  }
+]

data/redirects/f021395f-fd63-46cd-ab95-037c6f051730.json

@@ -0,0 +1,7 @@
+[
+  {
+    "bookId": "f021395f-fd63-46cd-ab95-037c6f051730",
+    "pageId": "2b4014ea-82b6-49ea-8c26-47ed38be3f34",
+    "pathname": "/books/precalculus-2e/pages/7-1-solving-trigonometric-equations-with-identities"
+  }
+]

script/prerender/constants.ts

@@ -0,0 +1,8 @@
+import config from '../../src/config';
+import { assertDefined } from '../../src/app/utils';
+
+export const RELEASE_ID = assertDefined(config.RELEASE_ID, 'REACT_APP_RELEASE_ID environment variable must be set');
+export const BUCKET_NAME = process.env.BUCKET_NAME || 'sandbox-unified-web-primary';
+export const BUCKET_REGION = process.env.BUCKET_REGION || 'us-east-1';
+export const PUBLIC_URL = process.env.PUBLIC_URL || `/rex/releases/${RELEASE_ID}`;
+export const WORK_REGION = process.env.WORK_REGION || 'us-east-2';

script/prerender/contentManifest.ts

@@ -0,0 +1,60 @@
+import { BookWithOSWebData, ArchiveTreeNode, ArchiveTree } from '../../src/app/content/types';
+import { content } from '../../src/app/content/routes';
+import { writeAssetFile } from './fileUtils';
+import { stripIdVersion } from '../../src/app/content/utils/idUtils';
+import { splitTitleParts } from '../../src/app/content/utils/archiveTreeUtils';
+
+const quoteValue = (value?: string) => value ? `"${value.replace(/"/g, '""')}"` : '""';
+
+export const renderAndSaveContentManifest = async(
+  saveFile: (path: string, contents: string) => Promise<unknown>,
+  books: BookWithOSWebData[]
+) => {
+
+  const rows = books.map(book => getContentsRows(book, book.tree))
+    .reduce((result, item) => ([...result, ...item]), [] as string[][]);
+
+  const manifestText = [
+    ['id', 'title', 'text title', 'language', 'slug', 'url', 'toc type', 'toc target type'],
+    ...rows,
+  ].map(row => row.map(quoteValue).join(',')).join('\n');
+
+  await saveFile('/rex/content-metadata.csv', manifestText);
+};
+
+function getContentsRows(
+  book: BookWithOSWebData,
+  node: ArchiveTree | ArchiveTreeNode,
+  chapterNumber?: string
+): string[][] {
+  const {title, toc_target_type} = node;
+  const [titleNumber, titleString] = splitTitleParts(node.title);
+  const textTitle = `${titleNumber || chapterNumber || ''} ${titleString}`.replace(/\s+/, ' ').trim();
+  const id = stripIdVersion(node.id);
+  const tocType = node.toc_type ?? (id === book.id ? 'book' : '');
+
+  const urlParams = tocType === 'book'
+    ? [node.slug, '']
+    : 'contents' in node
+      ? ['', '']
+      : [node.slug, content.getUrl({book: {slug: book.slug}, page: {slug: node.slug}})];
+
+  const contents = 'contents' in node
+    ? node.contents.map(child => getContentsRows(book, child, titleNumber || chapterNumber))
+      .reduce((result, item) => ([...result, ...item]), [] as string[][])
+    : [];
+
+  return [
+    [stripIdVersion(id), title, textTitle, book.language, ...urlParams, tocType, toc_target_type ?? ''],
+    ...contents,
+  ];
+}
+
+
+// simple helper for local
+const writeAssetFileAsync = async(filepath: string, contents: string) => {
+  return writeAssetFile(filepath, contents);
+};
+export const renderContentManifest = async(books: BookWithOSWebData[]) => {
+  return renderAndSaveContentManifest(writeAssetFileAsync, books);
+};

script/prerender/fileUtils.ts

@@ -5,9 +5,9 @@ import { fromContainerMetadata } from '@aws-sdk/credential-providers';
 import flow from 'lodash/fp/flow';
 import once from 'lodash/once';
 import path from 'path';
-import { assertDefined } from '../../src/app/utils';
 import createCache, { Cache } from '../../src/helpers/createCache';
 import { directoryExists, readFile, writeFile } from '../../src/helpers/fileUtils';
+import { BUCKET_REGION, PUBLIC_URL, BUCKET_NAME } from './constants';
 
 const ASSET_DIR = path.resolve(__dirname, '../../build');
 const CACHE_DIR = path.resolve(__dirname, '../../cache');
@@ -51,7 +51,7 @@ export const createDiskCache = <K extends string, V>(prefix: string): Cache<K, V
 
 // Generates a release path for a file without a leading /, used when uploading the release to S3
 function prefixReleasePath(filepath: string) {
-  let basePath = assertDefined(process.env.PUBLIC_URL, 'PUBLIC_URL environment variable not set');
+  let basePath = PUBLIC_URL;
   if (basePath[0] === '/') { basePath = basePath.slice(1); }
   return `${basePath}${filepath}`;
 }
@@ -61,7 +61,7 @@ const getS3Client = once(async() => {
   const credentials = await fromContainerMetadata();
 
   console.log('Initializing S3 client');
-  return new S3Client({ credentials, region: process.env.BUCKET_REGION });
+  return new S3Client({ credentials, region: BUCKET_REGION });
 });
 
 async function writeS3File(key: string, contents: string, contentType: string) {
@@ -70,7 +70,7 @@ async function writeS3File(key: string, contents: string, contentType: string) {
   console.log(`Writing s3 file: /${key}`);
   return s3Client.send(new PutObjectCommand({
     Body: contents,
-    Bucket: process.env.BUCKET_NAME,
+    Bucket: BUCKET_NAME,
     CacheControl: 'max-age=0',
     ContentType: contentType,
     Key: key,

script/prerender/fleet.ts

@@ -36,28 +36,21 @@ import path from 'path';
 import asyncPool from 'tiny-async-pool';
 import { makeUnifiedBookLoader } from '../../src/app/content/utils';
 import { assertDefined } from '../../src/app/utils';
-import config from '../../src/config';
 import BOOKS from '../../src/config.books';
 import createArchiveLoader from '../../src/gateways/createArchiveLoader';
 import { getBooksConfigSync } from '../../src/gateways/createBookConfigLoader';
 import createOSWebLoader from '../../src/gateways/createOSWebLoader';
 import { readFile } from '../../src/helpers/fileUtils';
 import { globalMinuteCounter, prepareBookPages } from './contentPages';
-import { SerializedBookMatch, SerializedPageMatch } from './contentRoutes';
+import { SerializedPageMatch } from './contentRoutes';
 import createRedirects from './createRedirects';
 import './logUnhandledRejectionsAndExit';
 import renderManifest from './renderManifest';
-import { SitemapPayload } from './sitemap';
-
-const {
-  ARCHIVE_URL,
-  CODE_VERSION,
-  OS_WEB_URL,
-  REACT_APP_OS_WEB_API_URL,
-  RELEASE_ID,
-} = config;
-
-assertDefined(RELEASE_ID, 'REACT_APP_RELEASE_ID environment variable must be set');
+import { SitemapPayload, renderAndSaveSitemapIndex } from './sitemap';
+import { writeS3ReleaseXmlFile } from './fileUtils';
+import { renderAndSaveContentManifest } from './contentManifest';
+import { ARCHIVE_URL, OS_WEB_URL, REACT_APP_OS_WEB_API_URL, CODE_VERSION } from '../../src/config';
+import { RELEASE_ID, WORK_REGION, BUCKET_NAME, BUCKET_REGION, PUBLIC_URL } from './constants';
 
 // Increasing this too much can lead to connection issues and greater memory usage in the manager
 const MAX_CONCURRENT_BOOKS = 5;
@@ -73,11 +66,6 @@ const PRERENDER_TIMEOUT_SECONDS = 3600;
 const WORKERS_STACK_CREATE_TIMEOUT_SECONDS = 300;
 const WORKERS_STACK_DELETE_TIMEOUT_SECONDS = WORKERS_STACK_CREATE_TIMEOUT_SECONDS;
 
-const BUCKET_NAME = process.env.BUCKET_NAME || 'sandbox-unified-web-primary';
-const BUCKET_REGION = process.env.BUCKET_REGION || 'us-east-1';
-const PUBLIC_URL = process.env.PUBLIC_URL || `/rex/releases/${RELEASE_ID}`;
-const WORK_REGION = process.env.WORK_REGION || 'us-east-2';
-
 // Docker does not accept forward slashes in the image tag
 const IMAGE_TAG = process.env.IMAGE_TAG || `${RELEASE_ID.replace(/\//g, '-')}`;
 
@@ -86,7 +74,6 @@ const sqsClient = new SQSClient({ region: WORK_REGION });
 
 type PageTask = { payload: SerializedPageMatch, type: 'page' };
 type SitemapTask = { payload: SitemapPayload, type: 'sitemap' };
-type SitemapIndexTask = { payload: SerializedBookMatch[], type: 'sitemapIndex' };
 
 const booksConfig = getBooksConfigSync();
 const archiveLoader = createArchiveLoader({
@@ -288,8 +275,7 @@ async function getQueueUrls(workersStackName: string) {
 class Stats {
   public pages = 0;
   public sitemaps = 0;
-  public sitemapIndexes = 0;
-  get total() { return this.pages + this.sitemaps + this.sitemapIndexes; }
+  get total() { return this.pages + this.sitemaps; }
 }
 
 function makePrepareAndQueueBook(workQueueUrl: string, stats: Stats) {
@@ -347,11 +333,7 @@ function makePrepareAndQueueBook(workQueueUrl: string, stats: Stats) {
 
     console.log(`[${book.title}] Sitemap queued`);
 
-    // Used in the sitemap index
-    return {
-      params: { book: { slug: book.slug } },
-      state: { bookUid: book.id, bookVersion: book.version },
-    };
+    return book;
   };
 }
 
@@ -371,14 +353,10 @@ async function queueWork(workQueueUrl: string) {
     `All ${stats.pages} page prerendering jobs and all ${stats.sitemaps} sitemap jobs queued`
   );
 
-  await sendWithRetries(sqsClient, new SendMessageCommand({
-    MessageBody: JSON.stringify({ payload: books, type: 'sitemapIndex' } as SitemapIndexTask),
-    QueueUrl: workQueueUrl,
-  }));
-
-  stats.sitemapIndexes = 1;
-
-  console.log('1 sitemap index job queued');
+  await Promise.all([
+    renderAndSaveSitemapIndex(writeS3ReleaseXmlFile, books),
+    renderAndSaveContentManifest(writeS3ReleaseXmlFile, books),
+  ]);
 
   return stats;
 }
@@ -463,8 +441,8 @@ async function finishRendering(stats: Stats) {
   const elapsedMinutes = globalMinuteCounter();
 
   console.log(
-    `Prerender complete in ${elapsedMinutes} minutes. Rendered ${stats.pages} pages, ${
-    stats.sitemaps} sitemaps and ${stats.sitemapIndexes} sitemap index. ${
+    `Prerender complete in ${elapsedMinutes} minutes. Rendered ${stats.pages} pages, and ${
+    stats.sitemaps} sitemaps. ${
     stats.total / elapsedMinutes}ppm`
   );
 }

script/prerender/local.ts

@@ -5,8 +5,6 @@ import { ArchivePage, VersionedArchiveBookWithConfig } from '../../src/app/conte
 import config from '../../src/config';
 import createArchiveLoader from '../../src/gateways/createArchiveLoader';
 import createBookConfigLoader from '../../src/gateways/createBookConfigLoader';
-import createBuyPrintConfigLoader from '../../src/gateways/createBuyPrintConfigLoader';
-import { BuyPrintResponse } from '../../src/gateways/createBuyPrintConfigLoader';
 import createHighlightClient from '../../src/gateways/createHighlightClient';
 import createOSWebLoader from '../../src/gateways/createOSWebLoader';
 import { OSWebBook } from '../../src/gateways/createOSWebLoader';
@@ -27,9 +25,9 @@ import { createDiskCache } from './fileUtils';
 import renderManifest from './renderManifest';
 import { renderSitemap, renderSitemapIndex } from './sitemap';
 import userLoader from './stubbedUserLoader';
+import { renderContentManifest } from './contentManifest';
 
 const {
-  REACT_APP_BUY_PRINT_CONFIG_URL,
   REACT_APP_HIGHLIGHTS_URL,
   REACT_APP_OS_WEB_API_URL,
   REACT_APP_SEARCH_URL,
@@ -59,17 +57,13 @@ async function render() {
   });
   const searchClient = createSearchClient(`http://localhost:${port}${REACT_APP_SEARCH_URL}`);
   const highlightClient = createHighlightClient(`http://localhost:${port}${REACT_APP_HIGHLIGHTS_URL}`);
-  const buyPrintConfigLoader = createBuyPrintConfigLoader(REACT_APP_BUY_PRINT_CONFIG_URL, {
-    cache: createDiskCache<string, BuyPrintResponse>('buy-print'),
-  });
   const practiceQuestionsLoader = createPracticeQuestionsLoader();
   const bookConfigLoader = createBookConfigLoader();
 
   const {server} = await startServer({port, onlyProxy: true});
   const renderHelpers = {
     archiveLoader,
     bookConfigLoader,
-    buyPrintConfigLoader,
     config,
     highlightClient,
     osWebLoader,
@@ -88,7 +82,8 @@ async function render() {
     await renderSitemap(book.slug, sitemap);
   }
 
-  await renderSitemapIndex();
+  await renderSitemapIndex(books);
+  await renderContentManifest(books);
   await renderManifest();
   await createRedirects(archiveLoader, osWebLoader);
 

script/prerender/sitemap.ts

@@ -1,12 +1,8 @@
-import filter from 'lodash/fp/filter';
-import flow from 'lodash/fp/flow';
-import get from 'lodash/fp/get';
-import identity from 'lodash/fp/identity';
-import map from 'lodash/fp/map';
-import max from 'lodash/fp/max';
 import sitemap, { SitemapItemOptions } from 'sitemap';
 import { SerializedPageMatch } from './contentRoutes';
 import { writeAssetFile } from './fileUtils';
+import { BookWithOSWebData } from '../../src/app/content/types';
+import { getSitemapItemOptions } from './contentPages';
 
 export const sitemapPath = (pathName: string) => `/rex/sitemaps/${pathName}.xml`;
 
@@ -28,40 +24,27 @@ export const renderAndSaveSitemap = async(
 
 export const renderAndSaveSitemapIndex = async(
   saveFile: (path: string, contents: string) => Promise<unknown>,
-  urls: SitemapItemOptions[]
+  books: BookWithOSWebData[]
 ) => {
-  const sitemapIndex = sitemap.buildSitemapIndex({ urls });
+  const sitemapIndex = sitemap.buildSitemapIndex({urls: books.map(book =>
+    getSitemapItemOptions(book, `https://openstax.org${sitemapPath(book.slug)}`)
+  )});
 
   const filePath = sitemapPath('index');
 
   await saveFile(filePath, sitemapIndex.toString());
-
-  return filePath;
 };
 
 // renderSitemap() and renderSitemapIndex() are used only by single-instance prerender code
 
-// Multi-instance code cannot store an array of sitemaps in memory and then use it across instances
-const sitemaps: SitemapItemOptions[] = [];
-
 const writeAssetFileAsync = async(filepath: string, contents: string) => {
   return writeAssetFile(filepath, contents);
 };
 
 export const renderSitemap = async(filename: string, urls: SitemapItemOptions[]) => {
-  const lastmod = flow(
-    map<SitemapItemOptions, (string | undefined)>(get('lastmod')),
-    filter<string | undefined>(identity),
-    max
-  )(urls);
-
-  const filePath = await renderAndSaveSitemap(writeAssetFileAsync, filename, urls);
-
-  const url = `https://openstax.org${filePath}`;
-
-  sitemaps.push({url, lastmod});
+  await renderAndSaveSitemap(writeAssetFileAsync, filename, urls);
 };
 
-export const renderSitemapIndex = async() => {
-  return renderAndSaveSitemapIndex(writeAssetFileAsync, sitemaps);
+export const renderSitemapIndex = async(books: BookWithOSWebData[]) => {
+  return renderAndSaveSitemapIndex(writeAssetFileAsync, books);
 };