Upgrade to Yarn 4 (#8061)

* updated yarn to version 4

* update readme

* update docker iamge for CI

* create a tmp dir for yarn stuff during asset compilation

* CI?

* replace deprecated yarn --frozen-lock flag with --immutable

* CI?

* CI

* remove yarn caching from CI

* fix CI

* add PR feedback

* add yarn 4.4.1 to docker.dev image

Author: hotzenklotz

.circleci/config.yml

@@ -39,7 +39,7 @@ jobs:
             DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base
       - run:
           name: Prepare dependency folders
-          command: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier ~/.cache/yarn
+          command: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
       - restore_cache:
           name: Restore target cache
           keys:
@@ -50,14 +50,9 @@ jobs:
           keys:
             - sbt-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "project/Dependencies.scala" }}
             - sbt-cache-{{ checksum ".circleci/cache_version" }}-
-      - restore_cache:
-          name: Restore yarn cache
-          keys:
-            - yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }}
-            - yarn-cache-{{ checksum ".circleci/cache_version" }}-
       - run:
           name: Install frontend dependencies
-          command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --frozen-lockfile
+          command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
       - run:
           name: Assert unique evolution numbers
           command: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
@@ -110,12 +105,6 @@ jobs:
             - "~/.ivy2"
             - "~/.cache/coursier"
 
-      - save_cache:
-          name: Save yarn cache
-          key: yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }}
-          paths:
-            - "~/.cache/yarn"
-
       - run:
           name: Build webknossos docker image
           command: |
@@ -261,7 +250,7 @@ jobs:
       - run:
           name: Install dependencies and sleep at least 3min
           command: |
-            yarn install --frozen-lockfile &
+            yarn install --immutable &
             sleep 180 &
             wait
       - run:
@@ -300,7 +289,7 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            yarn install --frozen-lockfile
+            yarn install --immutable
 
       - run:
           name: Assert GZIP

.gitignore

@@ -9,29 +9,57 @@ velocity.log
 dist/*
 lib_managed/
 src_managed/
-project/boot/
-project/plugins/project/
 RUNNING_PID
 .cache
 .classpath
 .project
 .settings/
 .target/
-logs/
+./data/
 .history
-target/
-binarydata/
-userBinaryData/
-binaryData
-tracingData/
+.eslintcache
+*.vscode
+.bsp
+.env
+.bloop
+.metals
+metals.sbt
+
+
+# Webknossos
 backup/
-./data/
-fossildb-dev/
+binaryData
+binarydata/
 boot/
+conf/application.conf-e
+dev-db/
+docker-compose.override.yml
+fossildb-dev/
+!frontend/javascripts/test/snapshots/type-check/.gitkeep
+!frontend/javascripts/test/snapshots/debug-htmls/.gitkeep
+frontend/javascripts/test/snapshots/debug-htmls/*
+frontend/javascripts/test/snapshots/type-check/*
+logs/
+project/boot/
+project/plugins/project/
+public-test/test-bundle
+public/bundle
+public/commit.txt
+public/docs
+public/docs/frontend-api
+public/test-bundle
+src/main/scala/rebel.xml
+target/
+temp-webknossos-schema**
+tracingData/
+userBinaryData/
+
+# IntelliJ
 .idea/
 .tmp/
 *.iml
 .idea_modules/
+
 node_modules/
 *.db
 *.class
@@ -42,41 +70,34 @@ node_modules/
 lifttemplate.iml
 lifttemplate.ipr
 lifttemplate.iws
-src/main/scala/rebel.xml
 *.obj
 *.zip
 *.skp
 *.mtl
-public/bundle
-public/test-bundle
-public-test/test-bundle
-public/docs
-public/commit.txt
 /knowledge
 conf/github.conf
 tmp/
 errorShots/
 flow-coverage/
-public/docs/frontend-api
-dev-db/
-docker-compose.override.yml
-frontend/javascripts/test/snapshots/debug-htmls/*
-!frontend/javascripts/test/snapshots/debug-htmls/.gitkeep
-frontend/javascripts/test/snapshots/type-check/*
-!frontend/javascripts/test/snapshots/type-check/.gitkeep
-.eslintcache
+
+
+# Sublime 
 *.sublime-project
 *.sublime-workspace
-**/screenshots/*.diff.png
-**/screenshots/*.new.png
+
+# Testing
 .nyc_output
 coverage
 coverage-ts
-*.vscode
-.bsp
-temp-webknossos-schema**
-conf/application.conf-e
-.env
-.bloop
-.metals
-metals.sbt
+**/screenshots/*.diff.png
+**/screenshots/*.new.png
+
+# Yarn 
+.yarn/*
+!.yarn/cache
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+tools/proxy/.yarn/*

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

CHANGELOG.unreleased.md

@@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Changed
 - Clicking on a bounding box within the bounding box tab centers it within the viewports and focusses it in the list. [#8049](https://github.com/scalableminds/webknossos/pull/8049)
 - For self-hosted versions, the text in the data set upload view was updated to recommend switching to webknossos.org. [#7996](https://github.com/scalableminds/webknossos/pull/7996)
+- Updated frontend package management to yarn version 4. [8061](https://github.com/scalableminds/webknossos/pull/8061)
 
 ### Fixed
 

Dockerfile.dev

@@ -11,18 +11,13 @@ RUN apt-get update && apt-get install apt-transport-https
 # add node package source
 RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash -
 
-# add yarn package source
-RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \
-  && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
-
 # Install sbt, node & build-essentials
 RUN apt-get update \
   && apt-get install -y \
   build-essential \
   findutils \
   nodejs \
   postgresql-client \
-  yarn \
   cmake \
   git \
   libdraco-dev \
@@ -32,3 +27,6 @@ RUN apt-get update \
   mesa-utils xvfb libgl1-mesa-dri libglapi-mesa libosmesa6 pkg-config x11proto-xext-dev xserver-xorg-dev libxext-dev libxi-dev \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Enable corepack for Node (which will install yarn)
+RUN corepack enable && corepack install --global yarn@4.4.1

package.json

@@ -73,7 +73,6 @@
     "webpack-cli": "^5.1.4",
     "webpack-dev-server": "^5.0.2"
   },
-  "optionalDependencies": {},
   "scripts": {
     "start": "node tools/proxy/proxy.js",
     "build": "node --max-old-space-size=4096 node_modules/.bin/webpack --env production",
@@ -213,12 +212,6 @@
     "url-join": "^4.0.0",
     "worker-loader": "^3.0.8"
   },
-  "resolutions": {
-    "**/mini-store": "^1.1.0",
-    "**/redux": "3.7.2",
-    "**/rc-resize-observer": "^1.4.0",
-    "**/rc-util": "^5.39.1"
-  },
   "ava": {
     "files": [
       "./public-test/test-bundle/**/*.{js,jsx}"
@@ -241,5 +234,6 @@
       "frontend/javascripts/test/**/*.*"
     ],
     "reporter": "lcov"
-  }
+  },
+  "packageManager": "yarn@4.4.1"
 }

project/AssetCompilation.scala

@@ -25,7 +25,7 @@ object AssetCompilation {
     try {
       val exitValue = startProcess(
         yarnPath.value,
-        List("install", "--frozen-lockfile"),
+        List("install", "--immutable"),
         baseDirectory.value
       ) ! streams.value.log
       if (exitValue != 0)
@@ -84,21 +84,6 @@ object AssetCompilation {
           streams.value.log.error("Could not test database entries to stage dir: " + e.getMessage)
       }
 
-      // copy node_modules for diff_schema.js
-      {
-        val nodeSrc = baseDirectory.value / "node_modules"
-        val nodeDest = target.value / "universal" / "stage" / "node_modules"
-        val tmpPath = baseDirectory.value / "tmp"
-        val streamsValue = streams.value.log
-
-        tmpPath.mkdirs
-        startProcess(yarnPath.value, List("init", "-y"), tmpPath) ! streamsValue
-        startProcess(yarnPath.value, List("add", (nodeSrc / "commander").getAbsolutePath), tmpPath) ! streamsValue
-        deleteRecursively(nodeDest)
-        copyRecursively(tmpPath / "node_modules", nodeDest)
-        deleteRecursively(tmpPath)
-      }
-
     } dependsOn yarnInstall
 
   private def slickClassesFromDBSchemaTask: Def.Initialize[Task[Seq[File]]] =