diff --git a/.dockerignore b/.dockerignore index b59e990a..f86eba4c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,12 +13,20 @@ **/*.jfm **/azds.yaml **/charts +!src/Analysim.Web/ClientApp/src/app/projects/project-file-explorer/csvdata-browser/visualize-data/charts +!src/Analysim.Web/ClientApp/src/app/projects/project-file-explorer/csvdata-browser/visualize-data/charts/** **/docker-compose* **/Dockerfile* **/node_modules **/npm-debug.log **/obj +**/appsettings*.json +!**/appsettings*.example.json +**/ClientApp/dist +**/ClientApp/.angular +**/ClientApp/src/assets/jupyter/dist +**/ClientApp/src/assets/jupyter/venv **/secrets.dev.yaml **/values.dev.yaml LICENSE -README.md \ No newline at end of file +README.md diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..5d298e5e --- /dev/null +++ b/.env.example @@ -0,0 +1,31 @@ +COMPOSE_PROJECT_NAME=analysim +ANALYSIM_DATA_ROOT=./data +ANALYSIM_DOMAIN=localhost +ANALYSIM_IMAGE_TAG=latest + +TLS_CERT_PATH=/etc/letsencrypt/live/localhost/fullchain.pem +TLS_KEY_PATH=/etc/letsencrypt/live/localhost/privkey.pem + +POSTGRES_DB=analysim +POSTGRES_USER=analysim +POSTGRES_PASSWORD=replace-with-a-long-random-password + +DB_CONNECTION_STRING=User ID=analysim;Password=replace-with-a-long-random-password;Server=postgres;Port=5432;Database=analysim;Pooling=true;SSL Mode=Require;Trust Server Certificate=true +JWT_ISSUER=AnalySim +JWT_AUDIENCE=https://localhost +JWT_SECRET=replace-with-a-long-random-secret +CLIENT_BASE_URL=https://localhost +ALLOWED_HOSTS=* +ADMIN_USER=ADMIN +REGISTRATION_CODE=replace-with-registration-code + +EMAIL_SERVER=smtp.example.com +EMAIL_PORT=587 +EMAIL_SENDER_NAME=no-reply-analysim +EMAIL_SENDER_EMAIL=no-reply@example.com +EMAIL_USERNAME=replace-with-email-username +EMAIL_PASSWORD=replace-with-email-password +USER_QUOTA=100000000 + +BACKUP_INTERVAL_SECONDS=86400 +BACKUP_RETENTION_DAYS=30 diff --git a/.github/workflows/backend-smoke-test.yml b/.github/workflows/backend-smoke-test.yml index d1f5cdbf..f734ad3e 100644 --- a/.github/workflows/backend-smoke-test.yml +++ b/.github/workflows/backend-smoke-test.yml @@ -9,6 +9,9 @@ on: jobs: backend-smoke-test: runs-on: ubuntu-latest + env: + BuildClientAppOnPublish: false + BuildJupyterLiteOnBuild: false steps: - name: Checkout repository diff --git a/.gitignore b/.gitignore index c4885ecd..3a70573d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.user *.userosscache *.sln.docstates +.env # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d37d3b3b..00000000 --- a/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Builds and runs ASPNET project Analysim -# Requires running database -FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -WORKDIR /source - -# install Node.js and Python -RUN apt-get update -yq && apt-get upgrade -yq && apt-get install -yq curl git nano python3 python3-venv python3-pip -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install -yq nodejs build-essential -RUN npm install -g npm - -# Install .Net EF Tools -RUN dotnet tool install --global dotnet-ef --version 6.0 -ENV PATH="$PATH:/root/.dotnet/tools" - -# then copy sources so previous step is cached -COPY src . - -# build project -WORKDIR Analysim.Web -RUN dotnet publish --configuration Release -o /app - -# run database migrations -FROM build as database-update -WORKDIR /source/Analysim.Web -CMD dotnet ef database update - -# run project in new container -FROM mcr.microsoft.com/dotnet/aspnet:6.0 as run -WORKDIR /app -COPY --from=build /app . - -#CMD ASPNETCORE_URLS=http://*:$PORT dotnet Analysim.Web.dll -CMD dotnet Analysim.Web.dll diff --git a/Dockerfile.run b/Dockerfile.run deleted file mode 100644 index 88ec924a..00000000 --- a/Dockerfile.run +++ /dev/null @@ -1,7 +0,0 @@ -# Run published project -FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base -WORKDIR /app -COPY src/Analysim.Web/bin/Release/net6.0/publish/ . - -#CMD ASPNETCORE_URLS=http://*:$PORT dotnet Analysim.Web.dll -CMD dotnet Analysim.Web.dll diff --git a/README.md b/README.md index 62a1a7b0..dd584743 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ git clone https://github.com/soft-eng-practicum/AnalySim.git - [Postman](https://www.postman.com/downloads/) for testing API calls - [Python](https://www.python.org/downloads/) for building project - [Docker](https://www.docker.com/products/docker-desktop) for testing deployment -- [Azure Data Studio](https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio?view=sql-server-ver15) or use online [Azure Portal](https://portal.azure.com) for browsing Azure Blob Storage ### Installing Angular package dependencies @@ -48,11 +47,13 @@ The AnalySim bridge is a JupyterLite/JupyterLab frontend extension that enables Analysim currently requires a SQL database (PostgreSQL) for both relational data and manual blob storage (Azure BlobStorage is no longer used / required). -In addition, an Outlook account is needed for email functionality. +In addition, an SMTP account is needed for email functionality. -All of these services are accessed via authentication information stored in the `appsettings.json` and `appsettings.Development.json` files which should be added under the `src/Analysim.Web` folder. +For local development, service credentials can be stored in +`src/Analysim.Web/appsettings.Development.json`. For Docker deployment, use +the root `.env` file created from `.env.example`. -The structure of the files are as follows (`XXX` means redacted): +The development settings file has this shape (`XXX` means redacted): ```json { @@ -68,7 +69,7 @@ The structure of the files are as follows (`XXX` means redacted): "DBConnectionString": "User ID=XXX;Password=XXX;Server=XXX;Port=5432;Database=XXX;Integrated Security=true;Pooling=true;SSL Mode=Require;Trust Server Certificate=true", }, "EmailSettings": { - "Server": "smtp-mail.outlook.com", + "Server": "smtp.example.com", "Port": 587, "SenderName": "no-reply-analysim", "SenderEmail": "XXX", @@ -96,7 +97,11 @@ The structure of the files are as follows (`XXX` means redacted): #### Adding admin users -Admin access in Analysim is controlled through the AdminUsers section of the `appsettings.json` and `appsettings.Development.json`. Each entry in the list corresponds to the username of a registered Analysim user. Admin users will see an Admin link in the navigation bar and can access the /admin section of the platform. To add or remove admin privileges, simply update this list and restart the server. +Admin access in Analysim is controlled by configured usernames. In local +development, set `AdminUsers` in `appsettings.Development.json`. In Docker +deployment, set the first admin user with `ADMIN_USER` in `.env`. Admin users +will see an Admin link in the navigation bar and can access the `/admin` +section of the platform. ⚠️ Important: The usernames must exactly match the usernames stored in the database (full uppercase eg. "ADMIN"). @@ -104,7 +109,9 @@ Admin access in Analysim is controlled through the AdminUsers section of the `ap Outlook no longer allows simple email authentication, so you must use another service that provides password authentication (e.g. Gmail). You can either use an existing account or create a new one and then fill in the `XXX` values under the section `EmailSettings` in the above file. -For email services, the correct BaseUrl is required within appsettings. For development, in `appsettings.Development.json` use: 'https://localhost:5001'. For deployment, in in `appsettings.json` the BaseUrl should match the necessary url. +For email links, the correct base URL is required. In local development, set +`ClientSettings:BaseUrl` in `appsettings.Development.json`. In Docker +deployment, set `CLIENT_BASE_URL` in `.env`. #### SQL database (also see Docker Compose option below) @@ -114,9 +121,9 @@ If you don't have a SQL database yet, download and install [PostgreSQL](https:// dotnet ef database update ``` -#### Azure Blob Storage +#### Blob storage -Blob storage is now replaced with the PostgreSQL database and no longer necessary. If you want to set it up regardless, follow these instructions. If you don't have an existing blob storage account, log into [Microsoft Azure](https://portal.azure.com), and create a ["Storage Account"](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview) with "Blob service" enabled. Then, select "Access Keys" on the left sidebar menu and copy one of the keys and insert both to replace the `XXX` in the `AzureStorageConnectionString` entry above. You will also need to insert your storage account name. In the same section on Azure, you can see the formatting for the correct Connection String as a guide. Blob storage falls under the [free student services](https://azure.microsoft.com/en-us/free/students/). +Blob storage is handled by PostgreSQL. Azure Blob Storage is no longer required. ### Running the project @@ -134,75 +141,75 @@ dotnet run --environment Development ## Deploying -The overall process of deployment is explained in the tutorial video on -[deploying ASP.Net on Heroku using Docker](https://www.youtube.com/watch?v=gQMT4al2Grg:). - -### Prerequisites -1. Download [Docker Desktop](https://www.docker.com/products/docker-desktop) -2. Download [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) -3. Enable Docker Support - -### Publish .Net project and create Docker image - -*Note:* Prepend `sudo` before each `docker` and `heroku` (except `dotnet`) command on Mac/Linux. - -### Using Docker Compose to compile and run the project by installing PostgreSQL in a container - -You can run Analysim and the PostGreSQL in containers using Docker Compose. You have to follow a 2-step process to first apply the database migrations: - -1. Build and run the migrations container: - ```bash - docker compose -f docker-compose.yml -f docker-compose-db.yml build db-update - docker compose -f docker-compose.yml -f docker-compose-db.yml run db-update - ``` -1. Run the Analysim process with the database: - ```bash - docker compose build - docker compose up - ``` - -### Using Docker manually to only run the project - -1. Publish *Analysim.Web* to the local folder (keep default location for folder), which can also be done on the command line: - ```bash - dotnet publish --configuration Release - ``` -1. Create the Docker image by running the following in the base project folder (e.g. `Analysim/`) : - ```bash - docker build -t analysim-dev -f Dockerfile-run . - ``` -1. Test image locally, by running it: - ```bash - docker run -it -p 127.0.0.1:80:80/tcp analysim-dev - ``` - You can test by opening a browser to http://localhost:80 (not https). - -### Register and upload Docker image to Heroku - -### Prerequisites -1. Docker setup (see above) -2. Download [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) - -*Note:* Prepend `sudo` before each `docker` and `heroku` (except `dotnet`) command on Mac/Linux. - -Run the following commands in terminal to update Heroku deployment ([more info](https://devcenter.heroku.com/articles/container-registry-and-runtime)): - -1. Login to Heroku and container service (if using `sudo`, you may need to copy-paste into browser): - ```bash - heroku login - heroku container:login - ``` -1. Tag the image name on Heroku's container registry: - ```bash - docker tag analysim-dev registry.heroku.com/analysim-dev - docker push registry.heroku.com/analysim-dev - ``` -1. Change to the `deploy/` folder and re-build image using Heroku CLI: - ```bash - cd deploy - heroku container:push web -a analysim-dev --context-path=.. - heroku container:release web -a analysim-dev - ``` +The production Docker deployment is defined by the root `docker-compose.yml`. +It runs PostgreSQL, a one-shot EF Core migration container, the ASP.NET Core +backend, an Nginx frontend/static server, scheduled PostgreSQL backups, and a +Certbot renewal container. + +Copy `.env.example` to `.env`, then update it for your environment. The `.env` +file is ignored by git and must not be committed. + +```sh +cp .env.example .env +``` + +Set `ANALYSIM_DATA_ROOT` in `.env` to a persistent folder. This folder stores +database data, backups, PostgreSQL certificates, and Certbot files. Create the +expected folders: + +```sh +export ANALYSIM_DATA_ROOT=/path/to/persistent/analysim-data +mkdir -p "$ANALYSIM_DATA_ROOT"/{postgres/data,postgres/certs,backups,certbot/conf,certbot/www} +``` + +At minimum, update these `.env` values before production: + +- `ANALYSIM_DATA_ROOT` +- `ANALYSIM_DOMAIN` +- `TLS_CERT_PATH` and `TLS_KEY_PATH` +- `POSTGRES_PASSWORD` +- `DB_CONNECTION_STRING` +- `JWT_SECRET` +- `CLIENT_BASE_URL` +- `ADMIN_USER` +- `REGISTRATION_CODE` +- `EMAIL_*` settings + +The backend image is designed to be safe to push to a registry because +production settings and secrets are supplied at runtime. + +The main stack starts in this order: + +```text +postgres -> migration -> backend -> nginx +``` + +Run the stack: + +```sh +docker compose --env-file .env build +docker compose --env-file .env up -d +``` + +Nginx is the public entry point on ports 80 and 443. It serves the Angular and +JupyterLite static files directly, proxies `/api/` to the backend, and uses +Let's Encrypt certificates mounted from `$ANALYSIM_DATA_ROOT/certbot`. + +For first-time certificate issuance, start with a valid DNS record for the +configured `ANALYSIM_DOMAIN`, ensure port 80 reaches the Nginx container, then +run Certbot with the same mounted webroot and config paths. Renewal is handled +by the `certbot` service. + +For local testing, self-signed certificates are enough. For production, use +Let's Encrypt or another trusted certificate provider. + +Backups are written to: + +```text +$ANALYSIM_DATA_ROOT/backups/postgres +``` + +Restore testing instructions are in `deploy/backup/restore-test.md`. ## Google Summer of Code application examples diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..b668ac0c --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,88 @@ +# AnalySim Docker Deployment + +The root `docker-compose.yml` is the main deployment entry point. + +## Services + +- `postgres`: PostgreSQL with externally mounted persistent data. +- `migration`: one-shot EF Core migration bundle. +- `backend`: ASP.NET Core API running Kestrel on the internal Docker network. +- `nginx`: public HTTP/HTTPS entry point, static Angular/JupyterLite server, and `/api/` reverse proxy. +- `backup`: scheduled PostgreSQL dumps with retention pruning. +- `certbot`: Let's Encrypt renewal loop using the Nginx webroot. + +## External State + +Copy `.env.example` to `.env`, then change the values for your deployment. +The `.env` file is ignored by git and must not be committed. + +Persistent state is controlled by: + +```text +ANALYSIM_DATA_ROOT=/path/to/persistent/analysim-data +``` + +If you run shell commands that use this value, export it too: + +```sh +export ANALYSIM_DATA_ROOT=/path/to/persistent/analysim-data +export ANALYSIM_DOMAIN=your-domain.example +``` + +Expected layout: + +```text +$ANALYSIM_DATA_ROOT/ + backups/ + certbot/ + conf/ + www/ + postgres/ + certs/ + server.crt + server.key + data/ +``` + +Change these before production: + +- `ANALYSIM_DATA_ROOT` +- `ANALYSIM_DOMAIN` +- `TLS_CERT_PATH` and `TLS_KEY_PATH` +- `POSTGRES_PASSWORD` and `DB_CONNECTION_STRING` +- `JWT_SECRET`, `CLIENT_BASE_URL`, `ADMIN_USER`, `REGISTRATION_CODE`, and email settings + +## First Run + +1. Point DNS for `ANALYSIM_DOMAIN` at the server. +2. Copy `.env.example` to `.env` and update the values. +3. Add PostgreSQL TLS files under `postgres/certs`. +4. Issue the first Let's Encrypt certificate using the same Certbot paths: + + ```sh + docker run --rm \ + -v "$ANALYSIM_DATA_ROOT/certbot/conf:/etc/letsencrypt" \ + -v "$ANALYSIM_DATA_ROOT/certbot/www:/var/www/certbot" \ + certbot/certbot certonly \ + --webroot \ + --webroot-path /var/www/certbot \ + -d "$ANALYSIM_DOMAIN" + ``` + +5. Start the stack: + + ```sh + docker compose up --build -d + ``` + +## Validation + +```sh +docker compose ps +docker compose logs migration +curl -I "https://$ANALYSIM_DOMAIN/" +curl "https://$ANALYSIM_DOMAIN/api/health" +``` + +Open a project notebook and confirm the JupyterLite iframe loads from +`/assets/jupyter/dist/lab/index.html`. diff --git a/deploy/backend/Dockerfile b/deploy/backend/Dockerfile new file mode 100644 index 00000000..f5441ad7 --- /dev/null +++ b/deploy/backend/Dockerfile @@ -0,0 +1,58 @@ +# syntax=docker/dockerfile:1 + +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS restore +WORKDIR /src + +COPY AnalySim.sln ./ +COPY src/Analysim.Core/Analysim.Core.csproj src/Analysim.Core/ +COPY src/Analysim.Infrastructure/Analysim.Infrastructure.csproj src/Analysim.Infrastructure/ +COPY src/Analysim.Web/Analysim.Web.csproj src/Analysim.Web/ +RUN dotnet restore src/Analysim.Web/Analysim.Web.csproj + +FROM restore AS build +COPY src ./src +RUN dotnet publish src/Analysim.Web/Analysim.Web.csproj \ + --configuration Release \ + --output /app/publish \ + --no-restore \ + /p:BuildClientAppOnPublish=false \ + /p:BuildJupyterLiteOnBuild=false +RUN find /app/publish -maxdepth 1 -name 'appsettings*.json' -delete + +FROM build AS migration-build +ENV ConnectionStrings__DBConnectionString="Host=localhost;Port=5432;Database=analysim;Username=analysim;Password=analysim" +ENV BuildClientAppOnPublish=false +ENV BuildJupyterLiteOnBuild=false +RUN dotnet tool install --tool-path /tools dotnet-ef --version 6.0.6 +RUN dotnet restore src/Analysim.Web/Analysim.Web.csproj --runtime linux-x64 +RUN dotnet build src/Analysim.Web/Analysim.Web.csproj \ + --configuration Release \ + --runtime linux-x64 \ + --no-self-contained \ + --no-restore \ + /p:BuildClientAppOnPublish=false \ + /p:BuildJupyterLiteOnBuild=false +RUN mkdir -p /app/migrations +RUN /tools/dotnet-ef migrations bundle \ + --project src/Analysim.Infrastructure/Analysim.Infrastructure.csproj \ + --startup-project src/Analysim.Web/Analysim.Web.csproj \ + --configuration Release \ + --runtime linux-x64 \ + --no-build \ + --output /app/migrations/efbundle + +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS runtime +WORKDIR /app +ENV ASPNETCORE_ENVIRONMENT=Production +ENV ASPNETCORE_URLS=http://+:8080 +ENV ClientApp__ServeFromBackend=false +EXPOSE 8080 +COPY --from=build /app/publish . +ENTRYPOINT ["dotnet", "Analysim.Web.dll"] + +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS migration +WORKDIR /app +ENV ASPNETCORE_ENVIRONMENT=Production +ENV ClientApp__ServeFromBackend=false +COPY --from=migration-build /app/migrations/efbundle ./efbundle +ENTRYPOINT ["./efbundle"] diff --git a/deploy/backup/Dockerfile b/deploy/backup/Dockerfile new file mode 100644 index 00000000..67e68017 --- /dev/null +++ b/deploy/backup/Dockerfile @@ -0,0 +1,6 @@ +FROM postgres:16-alpine + +COPY deploy/backup/backup.sh /usr/local/bin/analysim-backup +RUN chmod +x /usr/local/bin/analysim-backup + +ENTRYPOINT ["analysim-backup"] diff --git a/deploy/backup/backup.sh b/deploy/backup/backup.sh new file mode 100644 index 00000000..8d387821 --- /dev/null +++ b/deploy/backup/backup.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu + +: "${POSTGRES_HOST:=postgres}" +: "${POSTGRES_PORT:=5432}" +: "${POSTGRES_DB:?POSTGRES_DB is required}" +: "${POSTGRES_USER:?POSTGRES_USER is required}" +: "${BACKUP_DIR:=/backups/postgres}" +: "${BACKUP_INTERVAL_SECONDS:=86400}" +: "${BACKUP_RETENTION_DAYS:=30}" +: "${RUN_ONCE:=false}" + +if [ -n "${POSTGRES_PASSWORD_FILE:-}" ]; then + PGPASSWORD="$(cat "$POSTGRES_PASSWORD_FILE")" + export PGPASSWORD +fi + +if [ -z "${PGPASSWORD:-}" ] && [ -n "${POSTGRES_PASSWORD:-}" ]; then + PGPASSWORD="$POSTGRES_PASSWORD" + export PGPASSWORD +fi + +mkdir -p "$BACKUP_DIR" + +run_backup() { + timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + tmp_file="$BACKUP_DIR/analysim-$timestamp.dump.tmp" + final_file="$BACKUP_DIR/analysim-$timestamp.dump" + + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Starting PostgreSQL backup to $final_file" + pg_dump \ + --host "$POSTGRES_HOST" \ + --port "$POSTGRES_PORT" \ + --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" \ + --format custom \ + --file "$tmp_file" + + mv "$tmp_file" "$final_file" + size="$(du -h "$final_file" | awk '{print $1}')" + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Backup complete: $final_file ($size)" + + find "$BACKUP_DIR" -type f -name 'analysim-*.dump' -mtime +"$BACKUP_RETENTION_DAYS" -print -delete +} + +while true; do + run_backup + + if [ "$RUN_ONCE" = "true" ]; then + exit 0 + fi + + sleep "$BACKUP_INTERVAL_SECONDS" +done diff --git a/deploy/backup/restore-test.md b/deploy/backup/restore-test.md new file mode 100644 index 00000000..bf2159e9 --- /dev/null +++ b/deploy/backup/restore-test.md @@ -0,0 +1,17 @@ +# PostgreSQL Restore Test + +Use a disposable database for restore verification. Do not restore over production. + +```sh +createdb --host postgres --username "$POSTGRES_USER" analysim_restore_test +pg_restore \ + --host postgres \ + --username "$POSTGRES_USER" \ + --dbname analysim_restore_test \ + --clean \ + --if-exists \ + --no-owner \ + /backups/postgres/analysim-YYYYMMDDTHHMMSSZ.dump +``` + +After restore, run smoke queries against Identity tables, projects, notebooks, and blob content tables. Drop the disposable database after verification. diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile deleted file mode 100644 index 955c25d4..00000000 --- a/deploy/build.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Learn about building .NET container images: -# https://github.com/dotnet/dotnet-docker/blob/main/samples/README.md -FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -WORKDIR /source - -# copy csproj and restore as distinct layers -#COPY src/Analysim.Web/*.csproj . -#RUN dotnet restore --use-current-runtime - -# copy and publish app and libraries -COPY src/ . -WORKDIR /source/Analysim.Web -#RUN dotnet publish --use-current-runtime --self-contained false --no-restore -o /app --configuration Release - -# install NodeJS 18.x -# see https://github.com/nodesource/distributions/blob/master/README.md#deb -RUN apt-get update -yq -RUN apt-get install curl gnupg -yq -RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - &&\ - apt-get install -y nodejs - -# build app -RUN dotnet publish -o /app --configuration Release - -# problem: this needs to run when we bring up the docker stack, or maybe as a separate exec command -# see: https://codebuckets.com/2020/08/14/applying-entity-framework-migrations-to-a-docker-container/ -# or add to docker compose: https://itnext.io/database-development-in-docker-with-entity-framework-core-95772714626f -#RUN dotnet tool install --global dotnet-ef -#ENV PATH="$PATH:/root/.dotnet/tools" -#RUN dotnet ef database update - -# final stage/image -FROM mcr.microsoft.com/dotnet/aspnet:6.0 -WORKDIR /app -COPY --from=build /app . -ENTRYPOINT ["dotnet", "Analysim.Web.dll"] diff --git a/deploy/cert-renew b/deploy/cert-renew deleted file mode 100755 index fb3d8ad6..00000000 --- a/deploy/cert-renew +++ /dev/null @@ -1,10 +0,0 @@ -#! /bin/bash -CERT_DIR=`find /etc/letsencrypt/live -type d | tail -1` -# Currently must pause .Net server before doing this -http-server -p 80 /home/exouser/wwwroot/ & -sleep 3s -certbot renew -./convert-pem2pfx.sh $CERT_DIR -cp $CERT_DIR/fullchain.pfx ~/.aspnet/https/ -# kill the http server -kill $(jobs -p) diff --git a/deploy/convert-pem2pfx.sh b/deploy/convert-pem2pfx.sh deleted file mode 100755 index 3d4f6f07..00000000 --- a/deploy/convert-pem2pfx.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/bash -if [ "$1" == "" ]; then - echo "Usage: ./cert-renew " - echo "Example: ./cert-renew /etc/letsencrypt/live/analysim.tech/" - exit -1 -fi -openssl pkcs12 -inkey $1/privkey.pem -in $1/fullchain.pem -export -out $1/fullchain.pfx -password pass:analysim diff --git a/deploy/db_bup.sh b/deploy/db_bup.sh deleted file mode 100755 index d90c7c57..00000000 --- a/deploy/db_bup.sh +++ /dev/null @@ -1 +0,0 @@ -pg_dump -U analysim_dev -W -h localhost analysim_dev diff --git a/deploy/docker-run b/deploy/docker-run deleted file mode 100755 index 31709843..00000000 --- a/deploy/docker-run +++ /dev/null @@ -1 +0,0 @@ -docker run --rm -it -p 0.0.0.0:80:80 -p 0.0.0.0:443:443 --network="host" -e ASPNETCORE_URLS="https://0.0.0.0:443;http://0.0.0.0:80" -e ASPNETCORE_HTTPS_PORT=443 -e ASPNETCORE_Kestrel__Certificates__Default__Password="analysim" -e ASPNETCORE_Kestrel__Certificates__Default__Path=/https/fullchain.pfx -v ~/.aspnet/https:/https/ analysim-dev diff --git a/deploy/docker-run-nohttps b/deploy/docker-run-nohttps deleted file mode 100755 index fb6c65e6..00000000 --- a/deploy/docker-run-nohttps +++ /dev/null @@ -1 +0,0 @@ -docker run --rm -it -p 0.0.0.0:80:80 --network="host" -e ASPNETCORE_URLS="http://0.0.0.0:80" analysim-dev diff --git a/deploy/nginx/Dockerfile b/deploy/nginx/Dockerfile new file mode 100644 index 00000000..228ab981 --- /dev/null +++ b/deploy/nginx/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 + +FROM node:20-bookworm-slim AS build +WORKDIR /app/src/Analysim.Web/ClientApp + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + git \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +COPY src/Analysim.Web/ClientApp/package*.json ./ +RUN npm ci --legacy-peer-deps + +COPY src/Analysim.Web/ClientApp ./ +RUN npm run build:jupyterlite +RUN find src/assets/jupyter -mindepth 1 -maxdepth 1 ! -name dist -exec rm -rf {} + +RUN npm run build -- --configuration production + +FROM nginx:stable-alpine AS runtime +COPY deploy/nginx/nginx.conf /etc/nginx/nginx.conf +COPY deploy/nginx/conf.d/analysim.conf.template /etc/nginx/templates/default.conf.template +COPY --from=build /app/src/Analysim.Web/ClientApp/dist/ /usr/share/nginx/html/ +EXPOSE 80 443 diff --git a/deploy/nginx/conf.d/analysim.conf.template b/deploy/nginx/conf.d/analysim.conf.template new file mode 100644 index 00000000..9fe06fb9 --- /dev/null +++ b/deploy/nginx/conf.d/analysim.conf.template @@ -0,0 +1,56 @@ +upstream analysim_backend { + server backend:8080; +} + +server { + listen 80; + listen [::]:80; + server_name ${ANALYSIM_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + try_files $uri =404; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name ${ANALYSIM_DOMAIN}; + + ssl_certificate ${TLS_CERT_PATH}; + ssl_certificate_key ${TLS_KEY_PATH}; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_session_tickets off; + + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://analysim_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + + location ^~ /assets/jupyter/ { + try_files $uri =404; + add_header Cache-Control "public, max-age=3600"; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/deploy/nginx/nginx.conf b/deploy/nginx/nginx.conf new file mode 100644 index 00000000..1d8959ad --- /dev/null +++ b/deploy/nginx/nginx.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + types { + application/octet-stream whl; + application/x-ipynb+json ipynb; + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + client_max_body_size 100m; + + gzip on; + gzip_vary on; + gzip_types + application/javascript + application/json + application/wasm + image/svg+xml + text/css + text/plain; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/deploy/postgres/README.md b/deploy/postgres/README.md new file mode 100644 index 00000000..e40e6bce --- /dev/null +++ b/deploy/postgres/README.md @@ -0,0 +1,11 @@ +# PostgreSQL External Files + +Production PostgreSQL data and certificates should live outside the repository: + +```text +$ANALYSIM_DATA_ROOT/postgres/data +$ANALYSIM_DATA_ROOT/postgres/certs/server.crt +$ANALYSIM_DATA_ROOT/postgres/certs/server.key +``` + +The private key must be readable by the `postgres` user inside the container and should not be world-readable. diff --git a/deploy/postgres/pg_hba.conf b/deploy/postgres/pg_hba.conf new file mode 100644 index 00000000..0387d0d4 --- /dev/null +++ b/deploy/postgres/pg_hba.conf @@ -0,0 +1,2 @@ +local all all trust +hostssl all all all scram-sha-256 diff --git a/deploy/postgres/postgresql.conf b/deploy/postgres/postgresql.conf new file mode 100644 index 00000000..9decfe93 --- /dev/null +++ b/deploy/postgres/postgresql.conf @@ -0,0 +1,17 @@ +listen_addresses = '*' +port = 5432 +max_connections = 100 + +ssl = on +ssl_cert_file = '/etc/postgresql/certs/server.crt' +ssl_key_file = '/etc/postgresql/certs/server.key' + +shared_buffers = 256MB +effective_cache_size = 768MB +maintenance_work_mem = 64MB +checkpoint_completion_target = 0.9 +wal_buffers = 16MB +default_statistics_target = 100 + +log_timezone = 'UTC' +timezone = 'UTC' diff --git a/docker-compose-db.yml b/docker-compose-db.yml deleted file mode 100644 index 811cb079..00000000 --- a/docker-compose-db.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - - db-update: - image: analysim-dev - build: - target: database-update - depends_on: - - db - tty: true diff --git a/docker-compose.yml b/docker-compose.yml index fe7f66e9..ccd1ac9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,27 +1,148 @@ -# Use postgres/example user/password credentials services: + postgres: + image: postgres:16 + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-analysim} + POSTGRES_USER: ${POSTGRES_USER:-analysim} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-replace-with-a-long-random-password} + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256" + command: + - postgres + - -c + - config_file=/etc/postgresql/postgresql.conf + - -c + - hba_file=/etc/postgresql/pg_hba.conf + volumes: + - ${ANALYSIM_DATA_ROOT:-./data}/postgres/data:/var/lib/postgresql/data + - ${ANALYSIM_DATA_ROOT:-./data}/postgres/certs:/etc/postgresql/certs:ro + - ./deploy/postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro + - ./deploy/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s - db: - image: postgres - restart: always + migration: + image: analysim-migration:${ANALYSIM_IMAGE_TAG:-latest} + build: + context: . + dockerfile: deploy/backend/Dockerfile + target: migration + restart: "no" environment: - POSTGRES_USER: analysim_dev - POSTGRES_PASSWORD: NeuroScience2023 + ASPNETCORE_ENVIRONMENT: Production + ClientApp__ServeFromBackend: "false" + AllowedHosts: ${ALLOWED_HOSTS:-*} + AdminUsers__0: ${ADMIN_USER:-ADMIN} + registrationCodes__0: ${REGISTRATION_CODE:-replace-with-registration-code} + ConnectionStrings__DBConnectionString: ${DB_CONNECTION_STRING:-User ID=analysim;Password=replace-with-a-long-random-password;Server=postgres;Port=5432;Database=analysim;Pooling=true;SSL Mode=Require;Trust Server Certificate=true} + JwtSettings__Issuer: ${JWT_ISSUER:-AnalySim} + JwtSettings__Audience: ${JWT_AUDIENCE:-https://localhost} + JwtSettings__Secret: ${JWT_SECRET:-replace-with-a-long-random-secret} + ClientSettings__BaseUrl: ${CLIENT_BASE_URL:-https://localhost} + EmailSettings__Server: ${EMAIL_SERVER:-smtp.example.com} + EmailSettings__Port: ${EMAIL_PORT:-587} + EmailSettings__SenderName: ${EMAIL_SENDER_NAME:-no-reply-analysim} + EmailSettings__SenderEmail: ${EMAIL_SENDER_EMAIL:-no-reply@example.com} + EmailSettings__Username: ${EMAIL_USERNAME:-replace-with-email-username} + EmailSettings__Password: ${EMAIL_PASSWORD:-replace-with-email-password} + UserQuota: ${USER_QUOTA:-100000000} + volumes: + - ${ANALYSIM_DATA_ROOT:-./data}/postgres/certs:/app/certs/postgres:ro + depends_on: + postgres: + condition: service_healthy - analysim: - image: analysim-dev + backend: + image: analysim-backend:${ANALYSIM_IMAGE_TAG:-latest} build: context: . - dockerfile: Dockerfile - target: run - #network_mode: "service:db" + dockerfile: deploy/backend/Dockerfile + target: runtime + restart: unless-stopped + environment: + ASPNETCORE_ENVIRONMENT: Production + ASPNETCORE_URLS: http://+:8080 + ClientApp__ServeFromBackend: "false" + AllowedHosts: ${ALLOWED_HOSTS:-*} + AdminUsers__0: ${ADMIN_USER:-ADMIN} + registrationCodes__0: ${REGISTRATION_CODE:-replace-with-registration-code} + ConnectionStrings__DBConnectionString: ${DB_CONNECTION_STRING:-User ID=analysim;Password=replace-with-a-long-random-password;Server=postgres;Port=5432;Database=analysim;Pooling=true;SSL Mode=Require;Trust Server Certificate=true} + JwtSettings__Issuer: ${JWT_ISSUER:-AnalySim} + JwtSettings__Audience: ${JWT_AUDIENCE:-https://localhost} + JwtSettings__Secret: ${JWT_SECRET:-replace-with-a-long-random-secret} + ClientSettings__BaseUrl: ${CLIENT_BASE_URL:-https://localhost} + EmailSettings__Server: ${EMAIL_SERVER:-smtp.example.com} + EmailSettings__Port: ${EMAIL_PORT:-587} + EmailSettings__SenderName: ${EMAIL_SENDER_NAME:-no-reply-analysim} + EmailSettings__SenderEmail: ${EMAIL_SENDER_EMAIL:-no-reply@example.com} + EmailSettings__Username: ${EMAIL_USERNAME:-replace-with-email-username} + EmailSettings__Password: ${EMAIL_PASSWORD:-replace-with-email-password} + UserQuota: ${USER_QUOTA:-100000000} + expose: + - "8080" + volumes: + - ${ANALYSIM_DATA_ROOT:-./data}/postgres/certs:/app/certs/postgres:ro depends_on: - - db - restart: always + migration: + condition: service_completed_successfully + + nginx: + image: analysim-nginx:${ANALYSIM_IMAGE_TAG:-latest} + build: + context: . + dockerfile: deploy/nginx/Dockerfile + restart: unless-stopped environment: - ASPNETCORE_URLS: "http://0.0.0.0:80" - tty: true + ANALYSIM_DOMAIN: ${ANALYSIM_DOMAIN:-localhost} + TLS_CERT_PATH: ${TLS_CERT_PATH:-/etc/letsencrypt/live/localhost/fullchain.pem} + TLS_KEY_PATH: ${TLS_KEY_PATH:-/etc/letsencrypt/live/localhost/privkey.pem} ports: - - 80:80 + - "80:80" + - "443:443" + volumes: + - ${ANALYSIM_DATA_ROOT:-./data}/certbot/conf:/etc/letsencrypt:ro + - ${ANALYSIM_DATA_ROOT:-./data}/certbot/www:/var/www/certbot:ro + depends_on: + backend: + condition: service_started + backup: + image: analysim-backup:${ANALYSIM_IMAGE_TAG:-latest} + build: + context: . + dockerfile: deploy/backup/Dockerfile + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-analysim} + POSTGRES_USER: ${POSTGRES_USER:-analysim} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-replace-with-a-long-random-password} + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + BACKUP_DIR: /backups/postgres + BACKUP_INTERVAL_SECONDS: ${BACKUP_INTERVAL_SECONDS:-86400} + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30} + volumes: + - ${ANALYSIM_DATA_ROOT:-./data}/backups:/backups + depends_on: + postgres: + condition: service_healthy + certbot: + image: certbot/certbot + restart: unless-stopped + volumes: + - ${ANALYSIM_DATA_ROOT:-./data}/certbot/conf:/etc/letsencrypt + - ${ANALYSIM_DATA_ROOT:-./data}/certbot/www:/var/www/certbot + entrypoint: /bin/sh + command: + - -c + - | + trap exit TERM + while :; do + certbot renew --webroot --webroot-path /var/www/certbot --quiet + sleep 12h & wait $${!} + done diff --git a/src/Analysim.Web/Analysim.Web.csproj b/src/Analysim.Web/Analysim.Web.csproj index e5b57b0e..4384009d 100644 --- a/src/Analysim.Web/Analysim.Web.csproj +++ b/src/Analysim.Web/Analysim.Web.csproj @@ -10,6 +10,8 @@ false + true + true Analysim.Web Analysim.Web fd3b0967-a370-46f3-9b1c-3a225fca1fa9 @@ -66,13 +68,13 @@ - + - + diff --git a/src/Analysim.Web/ClientApp/angular.json b/src/Analysim.Web/ClientApp/angular.json index bff0a6cd..02c38833 100644 --- a/src/Analysim.Web/ClientApp/angular.json +++ b/src/Analysim.Web/ClientApp/angular.json @@ -55,7 +55,7 @@ { "type": "anyComponentStyle", "maximumWarning": "2kb", - "maximumError": "12kb" + "maximumError": "16kb" } ], "fileReplacements": [ diff --git a/src/Analysim.Web/ClientApp/src/app/admin/components/notebooks/admin-notebook-item/admin-notebook-item-display/admin-notebook-item-display/admin-notebook-item-display.component.ts b/src/Analysim.Web/ClientApp/src/app/admin/components/notebooks/admin-notebook-item/admin-notebook-item-display/admin-notebook-item-display/admin-notebook-item-display.component.ts index 3b360563..618b0976 100644 --- a/src/Analysim.Web/ClientApp/src/app/admin/components/notebooks/admin-notebook-item/admin-notebook-item-display/admin-notebook-item-display/admin-notebook-item-display.component.ts +++ b/src/Analysim.Web/ClientApp/src/app/admin/components/notebooks/admin-notebook-item/admin-notebook-item-display/admin-notebook-item-display/admin-notebook-item-display.component.ts @@ -95,7 +95,7 @@ export class AdminNotebookItemDisplayComponent implements OnInit, OnDestroy { } loadNotebook() { - const url = '../../../../../../../assets/jupyter/dist/lab/index.html'; + const url = '/assets/jupyter/dist/lab/index.html'; this.jupyterFrameSrc = this.sanitizer.bypassSecurityTrustResourceUrl(url); this.projectService.getNotebookFile(this.notebook, this.version) diff --git a/src/Analysim.Web/ClientApp/src/app/projects/project-overview/project-overview-view/project-content/project-notebook-item/project-notebook-item-display/project-notebook-item-display.component.ts b/src/Analysim.Web/ClientApp/src/app/projects/project-overview/project-overview-view/project-content/project-notebook-item/project-notebook-item-display/project-notebook-item-display.component.ts index ee154b25..29c13d67 100644 --- a/src/Analysim.Web/ClientApp/src/app/projects/project-overview/project-overview-view/project-content/project-notebook-item/project-notebook-item-display/project-notebook-item-display.component.ts +++ b/src/Analysim.Web/ClientApp/src/app/projects/project-overview/project-overview-view/project-content/project-notebook-item/project-notebook-item-display/project-notebook-item-display.component.ts @@ -98,7 +98,7 @@ export class ProjectNotebookItemDisplayComponent implements OnInit, OnDestroy { } loadNotebook() { - const url = '../../../../../../../assets/jupyter/dist/lab/index.html'; + const url = '/assets/jupyter/dist/lab/index.html'; this.jupyterFrameSrc = this.sanitizer.bypassSecurityTrustResourceUrl(url); this.projectService.getNotebookFile(this.notebook, this.version) diff --git a/src/Analysim.Web/Startup.cs b/src/Analysim.Web/Startup.cs index 028ed82d..ea363285 100644 --- a/src/Analysim.Web/Startup.cs +++ b/src/Analysim.Web/Startup.cs @@ -90,6 +90,8 @@ public void ConfigureServices(IServiceCollection services) // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerManager logger) { + var serveClientAppFromBackend = Configuration.GetValue("ClientApp:ServeFromBackend", true); + if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); @@ -104,6 +106,15 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerM } app.ConfigureExceptionHandler(logger); + + var forwardedHeadersOptions = new ForwardedHeadersOptions + { + ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto + }; + forwardedHeadersOptions.KnownNetworks.Clear(); + forwardedHeadersOptions.KnownProxies.Clear(); + app.UseForwardedHeaders(forwardedHeadersOptions); + app.UseHttpsRedirection(); // Add MIME type for Jupyter Lite wheel files by creating a provider and add the .whl mapping @@ -112,7 +123,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerM app.UseStaticFiles(); - if (!env.IsDevelopment()) + if (!env.IsDevelopment() && serveClientAppFromBackend) { app.UseSpaStaticFiles(new StaticFileOptions { @@ -122,11 +133,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerM app.UseCors("CorsPolicy"); - app.UseForwardedHeaders(new ForwardedHeadersOptions - { - ForwardedHeaders = ForwardedHeaders.All - }); - app.UseRouting(); app.UseAuthentication(); @@ -140,19 +146,22 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerM ); }); - app.UseSpa(spa => + if (serveClientAppFromBackend) { - // To learn more about options for serving an Angular SPA from ASP.NET Core, - // see https://go.microsoft.com/fwlink/?linkid=864501 + app.UseSpa(spa => + { + // To learn more about options for serving an Angular SPA from ASP.NET Core, + // see https://go.microsoft.com/fwlink/?linkid=864501 - spa.Options.SourcePath = "ClientApp"; + spa.Options.SourcePath = "ClientApp"; - if (env.IsDevelopment()) - { - spa.UseAngularCliServer(npmScript: "start"); - } + if (env.IsDevelopment()) + { + spa.UseAngularCliServer(npmScript: "start"); + } - }); + }); + } } } }