add devcontainer support

.devcontainer/devcontainer.json

@@ -0,0 +1,19 @@
+{
+    "name": "Local Dev",
+    "build": {
+      "dockerfile": "../Dockerfile",
+      "context": "..",
+      // "target": "dev-env",
+    },
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace/richb-press,type=bind,consistency=default",
+    "mounts": [],
+    "workspaceFolder": "/workspace/richb-press",
+    "extensions": [
+      "MS-vsliveshare.vsliveshare-pack"
+    ],
+    "settings": {
+      "files.trimTrailingWhitespace": true,
+      "files.insertFinalNewline": true,
+      "terminal.integrated.scrollback": 100000
+    }
+  }

Dockerfile

@@ -152,7 +152,7 @@ ENV PATH=$PATH:/root/.nvm/versions/node/v$NODE_VERSION/bin/
 # -----------------------
 # Create Virtualenv
 # -----------------------
-
+ENV VENV_ROOT=/openstax-built/venv
 COPY            ./dockerfiles/build/build-system-venv.sh \
     $PROJECT_ROOT/dockerfiles/build/build-system-venv.sh
 RUN $PROJECT_ROOT/dockerfiles/build/build-system-venv.sh
@@ -310,18 +310,19 @@ COPY ./output-producer-resource $PROJECT_ROOT/output-producer-resource/
 WORKDIR $PROJECT_ROOT/output-producer-resource/
 
 RUN set -x \
-    && . $PROJECT_ROOT/venv/bin/activate \
+    && . $VENV_ROOT/bin/activate \
     && python3 setup.py sdist \
     && pip3 install dist/output-producer-resource-*.tar.gz
 
 RUN mkdir -p /opt/resource
 RUN set -x \
-    && . $PROJECT_ROOT/venv/bin/activate \
+    && . $VENV_ROOT/bin/activate \
     && for script in check in out; do ln -s $(which $script) /opt/resource/; done
 
 
-RUN mv /root/.nvm $PROJECT_ROOT/nvm
-ENV PATH=$PATH:$PROJECT_ROOT/nvm/versions/node/v$NODE_VERSION/bin/
+ENV NVM_ROOT=/openstax-built/nvm
+RUN mv /root/.nvm $NVM_ROOT
+ENV PATH=$PATH:$NVM_ROOT/versions/node/v$NODE_VERSION/bin/
 
 COPY --from=base-scratch / /
 
@@ -336,11 +337,11 @@ COPY --from=build-jo-stage /usr/local/bin/jo /usr/local/bin/jo
 COPY --from=build-kcov-stage /usr/local/bin/kcov* /usr/local/bin/
 COPY --from=build-kcov-stage /usr/local/share/doc/kcov /usr/local/share/doc/kcov
 
-COPY --from=build-xhtml-validator-stage $PROJECT_ROOT/xhtml-validator/build/libs/xhtml-validator.jar $PROJECT_ROOT/xhtml-validator/build/libs/xhtml-validator.jar
-COPY --from=build-mathify-stage $PROJECT_ROOT/mathify/ $PROJECT_ROOT/mathify/
+COPY --from=build-xhtml-validator-stage $PROJECT_ROOT/xhtml-validator/build/libs/xhtml-validator.jar /openstax-built/xhtml-validator.jar
+COPY --from=build-mathify-stage $PROJECT_ROOT/mathify/ /openstax-built/mathify/
 COPY --from=build-python-stage $BAKERY_SRC_ROOT/scripts $BAKERY_SRC_ROOT/scripts
 COPY --from=build-python-stage $BAKERY_SRC_ROOT/scripts/gdoc $BAKERY_SRC_ROOT/scripts/gdoc
-COPY --from=build-python-stage $PROJECT_ROOT/venv/ $PROJECT_ROOT/venv/
+COPY --from=build-python-stage $VENV_ROOT/ $VENV_ROOT/
 
 # Copy cnx-recipes styles
 COPY ./cnx-recipes/recipes/output/ $PROJECT_ROOT/cnx-recipes/recipes/output/
@@ -357,6 +358,7 @@ COPY ./dockerfiles/entrypointd.sh \
     /dockerfiles/
 
 COPY ./step-config.json $PROJECT_ROOT
+
 # ^^^^^^^^^^^^^^^ Dockerfile.common ^^^^^^^^^^^^^^^
 
 # vvvvvvvvvvv Dockerfile.cli-endmatter vvvvvvvvvvvv

Dockerfile.common

@@ -130,7 +130,7 @@ ENV PATH=$PATH:/root/.nvm/versions/node/v$NODE_VERSION/bin/
 # -----------------------
 # Create Virtualenv
 # -----------------------
-
+ENV VENV_ROOT=/openstax-built/venv
 COPY            ./dockerfiles/build/build-system-venv.sh \
     $PROJECT_ROOT/dockerfiles/build/build-system-venv.sh
 RUN $PROJECT_ROOT/dockerfiles/build/build-system-venv.sh
@@ -288,18 +288,19 @@ COPY ./output-producer-resource $PROJECT_ROOT/output-producer-resource/
 WORKDIR $PROJECT_ROOT/output-producer-resource/
 
 RUN set -x \
-    && . $PROJECT_ROOT/venv/bin/activate \
+    && . $VENV_ROOT/bin/activate \
     && python3 setup.py sdist \
     && pip3 install dist/output-producer-resource-*.tar.gz
 
 RUN mkdir -p /opt/resource
 RUN set -x \
-    && . $PROJECT_ROOT/venv/bin/activate \
+    && . $VENV_ROOT/bin/activate \
     && for script in check in out; do ln -s $(which $script) /opt/resource/; done
 
 
-RUN mv /root/.nvm $PROJECT_ROOT/nvm
-ENV PATH=$PATH:$PROJECT_ROOT/nvm/versions/node/v$NODE_VERSION/bin/
+ENV NVM_ROOT=/openstax-built/nvm
+RUN mv /root/.nvm $NVM_ROOT
+ENV PATH=$PATH:$NVM_ROOT/versions/node/v$NODE_VERSION/bin/
 
 COPY --from=base-scratch / /
 
@@ -314,11 +315,11 @@ COPY --from=build-jo-stage /usr/local/bin/jo /usr/local/bin/jo
 COPY --from=build-kcov-stage /usr/local/bin/kcov* /usr/local/bin/
 COPY --from=build-kcov-stage /usr/local/share/doc/kcov /usr/local/share/doc/kcov
 
-COPY --from=build-xhtml-validator-stage $PROJECT_ROOT/xhtml-validator/build/libs/xhtml-validator.jar $PROJECT_ROOT/xhtml-validator/build/libs/xhtml-validator.jar
-COPY --from=build-mathify-stage $PROJECT_ROOT/mathify/ $PROJECT_ROOT/mathify/
+COPY --from=build-xhtml-validator-stage $PROJECT_ROOT/xhtml-validator/build/libs/xhtml-validator.jar /openstax-built/xhtml-validator.jar
+COPY --from=build-mathify-stage $PROJECT_ROOT/mathify/ /openstax-built/mathify/
 COPY --from=build-python-stage $BAKERY_SRC_ROOT/scripts $BAKERY_SRC_ROOT/scripts
 COPY --from=build-python-stage $BAKERY_SRC_ROOT/scripts/gdoc $BAKERY_SRC_ROOT/scripts/gdoc
-COPY --from=build-python-stage $PROJECT_ROOT/venv/ $PROJECT_ROOT/venv/
+COPY --from=build-python-stage $VENV_ROOT/ $VENV_ROOT/
 
 # Copy cnx-recipes styles
 COPY ./cnx-recipes/recipes/output/ $PROJECT_ROOT/cnx-recipes/recipes/output/
@@ -334,4 +335,4 @@ COPY ./dockerfiles/entrypointd.sh \
     ./dockerfiles/docker-entrypoint-with-kcov.sh \
     /dockerfiles/
 
-COPY ./step-config.json $PROJECT_ROOT
+COPY ./step-config.json $PROJECT_ROOT

build-concourse/script/archive_check_feed.sh

@@ -1,3 +1,3 @@
 curl "$ABL_FILE_URL" -o book-feed.json
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 check-feed book-feed.json "$CODE_VERSION" "$WEB_QUEUE_STATE_S3_BUCKET" "$CODE_VERSION.web-hosting-archive-queue.json" "$MAX_BOOKS_PER_TICK" "archive-dist" "archive"

build-concourse/script/git_check_feed.sh

@@ -1,4 +1,4 @@
 curl "$ABL_FILE_URL" -o book-feed.json
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 check-feed book-feed.json "$CODE_VERSION" "$WEB_QUEUE_STATE_S3_BUCKET" "$CODE_VERSION.web-hosting-git-queue.json" "$MAX_BOOKS_PER_TICK" "git-dist" "git"

cli.env

@@ -2,33 +2,34 @@ PREVIEW_APP_URL_PREFIX=apps/archive-localdev
 CORGI_ARTIFACTS_S3_BUCKET=openstax-sandbox-cops-artifacts
 ARG_S3_BUCKET_NAME=openstax-sandbox-cops-artifacts
 
-LOCAL_ATTIC_DIR=_attic
-INPUT_SOURCE_DIR=_book-input
+LOCAL_ATTIC_DIR=./data/_attic
+INPUT_SOURCE_DIR=./data/_book-input
 
+DOCKERFILES_ROOT=/workspace/richb-press/dockerfiles
 ABL_FILE_URL=https://raw.githubusercontent.com/openstax/content-manager-approved-books/main/approved-book-list.json
 
-IO_BOOK=/data/book
-IO_ARCHIVE_FETCHED=/data/archive-fetched
-IO_ARCHIVE_BOOK=/data/archive-book
-IO_ARCHIVE_JSONIFIED=/data/archive-jsonified
-IO_ARCHIVE_UPLOAD=/data/archive-upload
-IO_ARCHIVE_GDOCIFIED=/data/archive-gdocified
-IO_ARCHIVE_DOCX=/data/archive-docx
+IO_BOOK=./data/book
+IO_ARCHIVE_FETCHED=./data/archive-fetched
+IO_ARCHIVE_BOOK=./data/archive-book
+IO_ARCHIVE_JSONIFIED=./data/archive-jsonified
+IO_ARCHIVE_UPLOAD=./data/archive-upload
+IO_ARCHIVE_GDOCIFIED=./data/archive-gdocified
+IO_ARCHIVE_DOCX=./data/archive-docx
 
-IO_RESOURCES=/data/resources
-IO_UNUSED_RESOURCES=/data/unused-resources
-IO_FETCHED=/data/fetched-book-group
-IO_FETCH_META=/data/fetched-metadata-group
-IO_ASSEMBLED=/data/assembled-book-group
-IO_ASSEMBLE_META=/data/assembled-metadata-group
-IO_BAKED=/data/baked-book-group
-IO_BAKE_META=/data/baked-metadata-group
-IO_LINKED=/data/linked-single
-IO_MATHIFIED=/data/mathified-single
-IO_REX_LINKED=/data/rex-linked
-IO_DISASSEMBLED=/data/disassembled-single
-IO_ARTIFACTS=/data/artifacts-single
-IO_DISASSEMBLE_LINKED=/data/disassembled-linked-single
-IO_JSONIFIED=/data/jsonified-single
-IO_S3_ARCHIVE_QUEUE=/data/s3-archive-queue
-IO_S3_GIT_QUEUE=/data/s3-git-queue
+IO_RESOURCES=./data/resources
+IO_UNUSED_RESOURCES=./data/unused-resources
+IO_FETCHED=./data/fetched-book-group
+IO_FETCH_META=./data/fetched-metadata-group
+IO_ASSEMBLED=./data/assembled-book-group
+IO_ASSEMBLE_META=./data/assembled-metadata-group
+IO_BAKED=./data/baked-book-group
+IO_BAKE_META=./data/baked-metadata-group
+IO_LINKED=./data/linked-single
+IO_MATHIFIED=./data/mathified-single
+IO_REX_LINKED=./data/rex-linked
+IO_DISASSEMBLED=./data/disassembled-single
+IO_ARTIFACTS=./data/artifacts-single
+IO_DISASSEMBLE_LINKED=./data/disassembled-linked-single
+IO_JSONIFIED=./data/jsonified-single
+IO_S3_ARCHIVE_QUEUE=./data/s3-archive-queue
+IO_S3_GIT_QUEUE=./data/s3-git-queue

dockerfiles/build/build-stage-bakery-3rdparty.bash

@@ -2,6 +2,6 @@
 set -e
 
 # shellcheck disable=SC1090
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 pip3 install -r $BAKERY_SRC_ROOT/scripts/requirements.txt

dockerfiles/build/build-stage-bakery-install.bash

@@ -2,7 +2,7 @@
 set -e
 
 # shellcheck disable=SC1090
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 pip3 install $BAKERY_SRC_ROOT/scripts/.
 npm --prefix $BAKERY_SRC_ROOT/scripts install --production $BAKERY_SRC_ROOT/scripts

dockerfiles/build/build-stage-easybake-3rdparty.bash

@@ -2,6 +2,6 @@
 set -e
 
 # shellcheck disable=SC1090
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 python3 -m pip install -r $PROJECT_ROOT/cnx-easybake/requirements/main.txt

dockerfiles/build/build-stage-easybake-install.bash

@@ -2,6 +2,6 @@
 set -e
 
 # shellcheck disable=SC1090
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 python3 -m pip install $PROJECT_ROOT/cnx-easybake/

dockerfiles/build/build-stage-neb-3rdparty.bash

@@ -2,7 +2,7 @@
 set -e
 
 # shellcheck disable=SC1090
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 pip3 install -U setuptools wheel
 pip3 install -r $PROJECT_ROOT/nebuchadnezzar/requirements/main.txt

dockerfiles/build/build-stage-neb-install.bash

@@ -2,6 +2,6 @@
 set -e
 
 # shellcheck disable=SC1090
-source $PROJECT_ROOT/venv/bin/activate
+source $VENV_ROOT/bin/activate
 
 pip3 install $PROJECT_ROOT/nebuchadnezzar/

dockerfiles/build/build-system-venv.sh

@@ -1,12 +1,14 @@
 #!/bin/sh
 set -e
 
+VENV_ROOT=${VENV_ROOT:-/openstax/venv}
+
 if [ ! -z "$GITPOD_HOST" ]; then
-    mkdir -p $PROJECT_ROOT/venv/bin
-    echo '# This is a no-op because we are running inside gitpod so no need for a virtualenv' > $PROJECT_ROOT/venv/bin/activate
-else 
-    python3 -m venv $PROJECT_ROOT/venv
+    mkdir -p $VENV_ROOT/bin
+    echo '# This is a no-op because we are running inside gitpod so no need for a virtualenv' > $VENV_ROOT/bin/activate
+else
+    python3 -m venv $VENV_ROOT
     # shellcheck disable=SC1090
-    . $PROJECT_ROOT/venv/bin/activate
+    . $VENV_ROOT/bin/activate
     pip3 install --no-cache-dir -U 'pip<20'
 fi

dockerfiles/docker-entrypoint.sh

@@ -35,14 +35,14 @@ try() { "$@" || die "ERROR: could not run [$*]$c_none" 112; }
 [[ $PROJECT_ROOT ]] || die "Environment variable PROJECT_ROOT was not set. It should be set inside the Dockerfile"
 
 STEP_CONFIG_FILE=${STEP_CONFIG_FILE:-$PROJECT_ROOT/step-config.json}
-DOCKERFILES_ROOT=${DOCKERFILES_ROOT:-/dockerfiles}
+DOCKERFILES_ROOT=${DOCKERFILES_ROOT:-$PROJECT_ROOT/dockerfiles}
 BAKERY_SCRIPTS_ROOT=${BAKERY_SCRIPTS_ROOT:-$PROJECT_ROOT/output-producer-service/bakery/src}
-PYTHON_VENV_ROOT=${PYTHON_VENV_ROOT:-$PROJECT_ROOT/venv}
+PYTHON_VENV_ROOT=${PYTHON_VENV_ROOT:-$VENV_ROOT}
 RECIPES_ROOT=${RECIPES_ROOT:-$PROJECT_ROOT/recipes}
-MATHIFY_ROOT=${MATHIFY_ROOT:-$PROJECT_ROOT/mathify}
+MATHIFY_ROOT=${MATHIFY_ROOT:-/openstax-built/mathify}
 CNX_RECIPES_RECIPES_ROOT=${CNX_RECIPES_RECIPES_ROOT:-$PROJECT_ROOT/cnx-recipes/recipes/output}
 CNX_RECIPES_STYLES_ROOT=${CNX_RECIPES_STYLES_ROOT:-$PROJECT_ROOT/cnx-recipes/styles/output}
-XHTML_VALIDATOR_ROOT=${XHTML_VALIDATOR_ROOT:-$PROJECT_ROOT/xhtml-validator/build/libs}
+XHTML_VALIDATOR_JAR=${XHTML_VALIDATOR_JAR:-/openstax-built/xhtml-validator.jar}
 
 source $PYTHON_VENV_ROOT/bin/activate
 
@@ -83,7 +83,7 @@ function do_xhtml_validate() {
     check=$3
     for xhtmlfile in $(find $dir_name -name "$file_pattern")
     do
-        java -cp $XHTML_VALIDATOR_ROOT/xhtml-validator.jar org.openstax.xml.Main - $check broken-link < "$xhtmlfile" || failure=true
+        java -cp $XHTML_VALIDATOR_JAR org.openstax.xml.Main - $check broken-link < "$xhtmlfile" || failure=true
     done
     if $failure; then
         exit 1 # LCOV_EXCL_LINE
@@ -150,7 +150,8 @@ function do_step() {
             ensure_arg version 'Specify repo/branch/tag/commit or archive collection version (e.g. latest)'
             ensure_arg archive_server 'Specify archive server (e.g. cnx.org)'
 
-            [[ -d $INPUT_SOURCE_DIR ]] || mkdir $INPUT_SOURCE_DIR
+            [[ ! $INPUT_SOURCE_DIR ]] && die "Bug: Missing environment variable INPUT_SOURCE_DIR"
+            [[ -d $INPUT_SOURCE_DIR ]] || mkdir -p $INPUT_SOURCE_DIR
 
             check_output_dir INPUT_SOURCE_DIR
 
@@ -170,12 +171,12 @@ function do_step() {
 
             check_input_dir INPUT_SOURCE_DIR
             check_output_dir IO_BOOK
-            
+
             tail $INPUT_SOURCE_DIR/*
             cp $INPUT_SOURCE_DIR/id $IO_BOOK/job_id
             cp $INPUT_SOURCE_DIR/version $IO_BOOK/version
             if [[ $recipe != "default" && $recipe != "*" ]]; then
-                cp $INPUT_SOURCE_DIR/collection_style $IO_BOOK/style 
+                cp $INPUT_SOURCE_DIR/collection_style $IO_BOOK/style
             fi
 
             # Detect if this is a git book or an archive book.
@@ -209,7 +210,7 @@ function do_step() {
                 pdf_filename="$(cat $IO_BOOK/collection_id)-$(cat $IO_BOOK/version)-$(cat $IO_BOOK/server_shortname)-$(cat $IO_BOOK/job_id).pdf"
                 echo "$pdf_filename" > $IO_BOOK/pdf_filename
             fi
-            
+
             return
         ;;
     esac
@@ -227,7 +228,7 @@ function do_step() {
 
         for required_env in $required_envs; do
             ensure_arg $(echo $required_env | tr -d "'")
-        done        
+        done
         for input_dir in $input_dirs; do
             check_input_dir $(echo $input_dir | tr -d "'")
         done
@@ -325,7 +326,7 @@ function simulate_dirs_after() {
 
             if [[ -d "$LOCAL_ATTIC_DIR/$io_name" ]]; then
                 try rm -rf "$LOCAL_ATTIC_DIR/$io_name"
-            fi 
+            fi
             try mv "$child_dir_path" "$LOCAL_ATTIC_DIR/$io_name"
         done
     fi
@@ -340,7 +341,7 @@ function simulate_dirs_after() {
             try rm -rf "$child_dir_path"
         done
     fi
- 
+
 }
 
 
@@ -355,7 +356,7 @@ elif [[ $pipeline_steps = 'null' ]]; then
 else
     do_step_named local-create-book-directory "${@:2}"
     do_step_named look-up-book
-    
+
     for step_name in $pipeline_steps; do
         do_step_named $(echo $step_name | tr -d "'")
     done

dockerfiles/steps/git-fetch-metadata.bash

@@ -21,6 +21,6 @@ fi
 try fetch-update-meta "$IO_FETCH_META/.git" "$IO_FETCH_META/modules" "$IO_FETCH_META/collections" "$ARG_GIT_REF" "$IO_FETCH_META/canonical.json"
 try rm -rf "$IO_FETCH_META/.git"
 
-try fetch-map-resources "$IO_FETCH_META/modules" "$IO_FETCH_META/media" . "$IO_UNUSED_RESOURCES"
+try fetch-map-resources "$IO_FETCH_META/modules" "$IO_FETCH_META/media" "$(cd $IO_RESOURCES && cd .. && pwd)" "$IO_UNUSED_RESOURCES"
 # Either the media is in resources or unused-resources, this folder should be empty (-d will fail otherwise)
 try rm -d "$IO_FETCH_META/media"

dockerfiles/steps/git-validate-xhtml.bash

@@ -2,6 +2,6 @@
 while IFS= read -r -d '' xhtmlfile
 do
     say "XHTML-validating $xhtmlfile"
-    try java -cp $XHTML_VALIDATOR_ROOT/xhtml-validator.jar org.openstax.xml.Main - duplicate-id broken-link < "$xhtmlfile"
+    try java -cp $XHTML_VALIDATOR_JAR org.openstax.xml.Main - duplicate-id broken-link < "$xhtmlfile"
 done <   <(find $IO_DISASSEMBLE_LINKED -name '*.xhtml' -print0)
 # LCOV_EXCL_STOP