Merge branch 'refactoring-for-extension' into 'main'

Base docker image off ucbears

See merge request lap/sftp_handler!8

Author: danschmidt5189

.gitignore

@@ -1,14 +1,13 @@
-# Convoluted gitkeep setting to keep the data directory
-# structure intact without tracking its contents.
+# secrets file
+.env
 
 vendor
 .bundle
 
+# Convoluted gitkeep setting to keep the data directory
+# structure intact without tracking its contents.
 data/*
 !data/.keep
 
 # Build/test artifacts
 artifacts/*
-
-secrets/*
-!secrets/.keep

Dockerfile

@@ -1,22 +1,95 @@
-#FROM debian:bullseye-slim
-FROM ruby:3.0
-USER root
+# =============================================================================
+# Target: base
+#
+# The base stage scaffolds elements which are common to building and running
+# the application, such as installing ca-certificates, creating the app user,
+# and installing runtime system dependencies.
+FROM ruby:3.0.3-slim AS base
+
+# ------------------------------------------------------------
+# Create the application user/group and installation directory
+
+# UCBEARS uses the "altmedia" user and group because (historical/permissions) reasons
+ENV APP_USER=alma
+ENV APP_UID=40054
+
+RUN groupadd --system --gid $APP_UID $APP_USER \
+    && useradd --home-dir /opt/app --system --uid $APP_UID --gid $APP_USER $APP_USER
+
+RUN mkdir -p /opt/app \
+    && chown -R $APP_USER:$APP_USER /opt/app /usr/local/bundle
 
-RUN apt-get update 
+# ------------------------------------------------------------
+# Install packages common to dev and prod.
 
-# Create the application user/group and application directory
-RUN groupadd -g 40054 alma && \
-    useradd -r -s /sbin/nologin -M -u 40054 -g alma alma && \
-    mkdir -p /opt/app && \
-    chown -R alma:alma /opt/app 
+# Install standard packages from the Debian repository
+RUN apt-get update -qq
 
-# Run everything else as the alma user
+# ------------------------------------------------------------
+# Run configuration
+
+# All subsequent commands are executed relative to this directory.
 WORKDIR /opt/app
 
-COPY --chown=alma Gemfile* ./
-RUN bundle install --system
-COPY --chown=alma . .
+# Run as the application user to minimize risk to the host.
+USER $APP_USER
 
-USER alma
+# Uses the get_gobi script as the entrypoint, so any arguments passed to `docker run`
+# at invocation are passed directly to this script.
 ENTRYPOINT ["ruby","/opt/app/lib/get_gobi.rb"]
-#CMD ["help"]
+
+# =============================================================================
+# Target: development
+#
+# The development stage installs build dependencies (system packages needed to
+# install all your gems) along with your bundle. It's "heavier" than the
+# production target.
+FROM base AS development
+
+# ------------------------------------------------------------
+# Install build packages
+
+# Temporarily switch back to root
+USER root
+
+# Install system packages needed to build gems with C extensions.
+RUN apt-get install -y --no-install-recommends \
+    g++ \
+    git \
+    make
+
+# ------------------------------------------------------------
+# Install Ruby gems
+
+# Drop back to $APP_USER.
+USER $APP_USER
+
+# Base image ships with an older version of bundler
+RUN gem install bundler --version 2.2.33
+
+# Install gems. We don't enforce the validity of the Gemfile.lock until the
+# final (production) stage.
+COPY --chown=$APP_USER:$APP_USER Gemfile* ./
+RUN bundle install
+
+# Copy the rest of the codebase. We do this after bundle-install so that
+# changes unrelated to the gemset don't invalidate the cache and force a slow
+# re-install.
+COPY --chown=$APP_USER:$APP_USER . .
+
+# =============================================================================
+# Target: production
+#
+# The production stage extends the base image with the application and gemset
+# built in the development stage. It includes runtime dependencies (including
+# test dependencies, due to quirks of our Jenkins build) but tries to minimize
+# heavyweight build dependencies.
+FROM base AS production
+
+# Copy the built codebase from the dev stage
+COPY --from=development --chown=$APP_USER /opt/app /opt/app
+COPY --from=development --chown=$APP_USER /usr/local/bundle /usr/local/bundle
+
+# Ensure the bundle is installed and the Gemfile.lock is synced.
+RUN bundle config set frozen 'true'
+RUN bundle install --local

Gemfile.lock

@@ -11,6 +11,7 @@ GEM
     diff-lcs (1.5.0)
     erb (2.2.3)
       cgi
+    getoptlong (0.1.1)
     hashdiff (1.0.1)
     net-sftp (3.0.0)
       net-ssh (>= 5.0.0, < 7.0.0)
@@ -35,6 +36,8 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
     rspec-support (3.11.0)
+    rspec_junit_formatter (0.5.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (1.26.0)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
@@ -46,6 +49,8 @@ GEM
       unicode-display_width (>= 1.4.0, < 3.0)
     rubocop-ast (1.16.0)
       parser (>= 3.1.1.0)
+    rubocop-checkstyle_formatter (0.5.0)
+      rubocop (>= 1.14.0)
     ruby-progressbar (1.11.0)
     thor (1.2.1)
     unicode-display_width (2.1.0)
@@ -61,9 +66,12 @@ PLATFORMS
 DEPENDENCIES
   colorize
   erb
+  getoptlong
   net-sftp
   rspec
+  rspec_junit_formatter
   rubocop
+  rubocop-checkstyle_formatter
   thor (~> 1.1)
   uri
   webmock

README.md

@@ -1,44 +1,70 @@
 # GOBI Processor
-A command-line tool for retrieving GOBI order marc files (.ord) via sftp. Default remote directory is gobiord and local directory is data.
 
-## Building the app
+A command-line tool for retrieving GOBI order marc files (.ord) via sftp.
 
-```sh
-docker-compose build
+## Running it locally
+
+To run the application for real, you'll need to configure a local `.env` file with the GOBI password, which you can obtain from LastPass (search for "gobi"). Don't worry: this file is git-ignored.
+
+```ini
+# ./.env (in the directory you cloned this from GitLab)
+GOBI_PASS=password-from-lastpass
 ```
 
-## Running it locally
+---
+
+> **Additional Configuration:** The app accepts a few other configuration options, which you can see in `config/connections.yml`. The default values should work fine, however, so you probably don't need to change them. Also note that everything is mocked in the rspec tests, where you're not actually hitting the GOBI server.
 
-Look for todays Gobi order file will be name eboo{mmdd}.ord on the gobi server in the gobiord directory.
-File by default will go into /opt/app/data directory
+---
+
+With that configured, you can run the application like so:
 
 ```sh
-docker-compose up 
+docker compose build # as always, you need to build it first
+docker compose run --rm app ebook0413.ord
 ```
 
-Look for a specific file
+Today's GOBI order file will be at `/gobiord/eboo{mmdd}.ord` on their server. By default, the app copies it into the `/opt/app/data` directory within the container (mapped to `./data` in development), but this can be overwritten by the `--local_dir` flag:
 
 ```sh
-docker-compose run --rm gobi_sftp ebook0413.ord 
+docker compose run --rm app --local_dir /path/in/container
 ```
 
-Look for a specific file with command line arguments
+Look for a specific file by passing its name as an argument:
 
 ```sh
-docker-compose run --rm gobi_sftp ebook0413.ord --remote_dir gobiord local_dir path_to_local_dir
+docker-compose run --rm app ebook0413.ord
 ```
 
+You can also pass additional arguments when looking for a specific file:
 
-## Running it in production
+```sh
+docker-compose run --rm app ebook0413.ord \
+    --remote_dir gobiord-staging \
+    --local_dir path_to_local_dir
+```
+
+## Testing
 
-Standard way this should be set to run each day to look for a file if it exists
+The app includes a number of RSpec tests integrated into its Jenkins pipeline. You can run the tests locally via:
 
 ```sh
-docker run --rm gobi_sftp --local_dir '/path_to_local_dir' 
+docker compose run --rm --entrypoint=rspec app
 ```
 
-Example if you wanted to specify a specific remote directory as well
+Or by shelling into the container and running RSpec from there:
 
 ```sh
-docker run --rm gobi_sftp ebook0312.ord --remote_dir remote_directory --local_dir '/path_to_local_dir' 
+docker compose run --rm --entrypoint=sh app
+rspec
 ```
+
+---
+
+> **Insecure KEX:** As of 04/28/22, Gobi's server only supports outdated Diffie-Hellman key exchange algorithms, which you local sftp client will probably (and correctly) refuse. If you want to test manually, then for now you must explicitly allow the old algorithm by passing the following option to your sftp client: `-oKexAlgorithms=+diffie-hellman-group1-sha1`.
+
+---
+
+## Production configuration
+
+In production, the app must copy the files to a place accessible to Alma, which is configured to pull them daily at 9PM. As of 04/28/22, this was decided to be the `gobi-ebook-eocr` directory in the `alma` NetApp volume.

config/connections.yml

@@ -1,4 +1,4 @@
 gobi:
-    host: <%= ENV['GOBI_HOST'] %> 
-    user: <%= ENV['GOBI_USER'] %>
-    password: <%= ENV['GOBI_PASS'] %>
+    host: <%= ENV.fetch('GOBI_HOST', 'ftp.ybp.com') %>
+    user: <%= ENV.fetch('GOBI_USER', 'berkeley') %>
+    password: <%= ENV.fetch('GOBI_PASS', 'p4$$w0r6') %>

docker-compose.yml

@@ -1,10 +1,13 @@
 services:
-  gobi_sftp:
-    build: .
+  app:
+    build:
+      context: .
+      target: development
+    environment:
+      GOBI_PASS: "${GOBI_PASS}"
     image: containers.lib.berkeley.edu/lap/gobi_sftp/development:latest
     init: yes
     volumes:
-      - ./:/opt/app:rw
-      - ./secrets:/run/secrets
+      - ./:/opt/app
 
 version: "3.8"