From d8ac2a4e21a998036ba26d24aef786dd9ff1a691 Mon Sep 17 00:00:00 2001 From: Alberto Casas Ortiz Date: Mon, 6 Jul 2026 18:27:35 -0700 Subject: [PATCH 1/5] Implemented deletion of S3 files when session or trial is removed. --- mcserver/models.py | 52 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/mcserver/models.py b/mcserver/models.py index e2731e9..e85ebbd 100644 --- a/mcserver/models.py +++ b/mcserver/models.py @@ -1,24 +1,25 @@ -import json - -from django.db import models -from django.contrib.auth.models import AbstractUser -from django.core.validators import MinValueValidator, MaxValueValidator import os import uuid -import base64 -import pathlib from http import HTTPStatus -from django.utils import timezone + +from django.contrib.auth.models import AbstractUser from django.core.exceptions import ValidationError -from django.db.models.signals import post_save -from django.contrib.auth.signals import user_logged_in +from django.db import models +from django.db.models.signals import post_save, pre_delete from django.dispatch import receiver -from rest_framework.authtoken.models import Token +from django.utils import timezone from django.utils.translation import gettext as _ -from rest_framework import status -from django.conf import settings +def delete_s3_file(file_field): + """Delete a file from the configured Django storage backend.""" + if not file_field or not file_field.name: + return + + try: + file_field.delete(save=False) + except Exception as e: + print(f"Error deleting file '{file_field.name}': {e}") def random_filename(instance, filename): return "{}-{}".format(uuid.uuid4(), filename) @@ -255,8 +256,7 @@ class ResetPassword(models.Model): datetime = models.DateField(default=timezone.now) from django_otp.plugins.otp_email.models import EmailDevice -from django.template.loader import render_to_string -from mcserver.customEmailDevice import CustomEmailDevice + @receiver(post_save, sender=User) def create_profile(sender, instance, created, **kwargs): @@ -572,3 +572,25 @@ def get_available_data(self, only_public=False, subject_id=None, share_token=Non subject_ids.append(subject.id) return data + +@receiver(pre_delete, sender=Video) +def delete_video_files(sender, instance, **kwargs): + """Delete Video files when the record is deleted.""" + delete_s3_file(instance.video) + delete_s3_file(instance.video_thumb) + delete_s3_file(instance.keypoints) + +@receiver(pre_delete, sender=Result) +def delete_result_files(sender, instance, **kwargs): + """Delete Result media when the record is deleted.""" + delete_s3_file(instance.media) + +@receiver(pre_delete, sender=Session) +def delete_session_files(sender, instance, **kwargs): + """Delete Session QR code when the record is deleted.""" + delete_s3_file(instance.qrcode) + +@receiver(pre_delete, sender=DownloadLog) +def delete_download_log_files(sender, instance, **kwargs): + """Delete DownloadLog archive when the record is deleted.""" + delete_s3_file(instance.media) \ No newline at end of file From f3da886247937beb4d12716b1dd40ca468341325 Mon Sep 17 00:00:00 2001 From: Alberto Casas Ortiz Date: Tue, 28 Jul 2026 12:13:41 -0700 Subject: [PATCH 2/5] Updated README.md --- README.md | 196 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 167 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 40b38d5..e1c3a18 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,31 @@ -# OpenCap API -## Workflow for app.opencap.ai +# ๐ŸŽฏ OpenCap API + +[![Python Version](https://img.shields.io/badge/python-3.7+-blue.svg)](https://www.python.org/downloads/) +[![Django Version](https://img.shields.io/badge/django-3.1.14+-green.svg)](https://www.djangoproject.com/) +[![License](https://img.shields.io/badge/license-APACHE_2.0-blue.svg)](LICENSE.md) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/opencap-org/opencap-api/pulls) + +> Backend API for OpenCap - The open-source platform for biomechanical motion capture and gait analysis using mobile devices. + +## ๐Ÿ“– Table of Contents + +- [Overview](#overview) +- [Workflow](#workflow) +- [Getting Started](#getting-started) +- [API Documentation](#api-documentation) +- [Development](#development) +- [Testing](#testing) +- [Internationalization](#internationalization) +- [Deployment](#deployment) +- [Contributing](#contributing) +- [License](#license) + +## ๐Ÿ”ญ Overview + +OpenCap is an open source biomechanical motion capture platform that leverages iOS devices to capture and analyze human movement. This repository contains the Django-based backend API that orchestrates the entire workflow, from session management to video processing and biomechanical analysis. + +## ๐Ÿ”„ Workflow + 1. User enters the website (app.opencap.ai) 2. The website calls the backend and creates a session 3. The session generates a QR code displayed in the webapp @@ -12,56 +38,168 @@ 10. Video processing pipeline pools sessions in 'processing' state and processes them 11. After processing, results are sent to the backend and the backend changes its state to 'done' -## Installation +## ๐Ÿš€ Getting Started + +### Prerequisites + +- Python 3.7+ +- [gettext](https://www.gnu.org/software/gettext/) (for internationalization) + +### Installation -Clone this repo, then: +1. Clone the repository: +```bash +git clone https://github.com/opencap-org/opencap-api.git +cd opencap-api ``` -conda create -n opencap python=3.7 + +2. Create and activate a conda environment: +```bash +conda create -n opencap python=3.7 conda activate opencap -pip install -r requirements.txt ``` -Create the `.env` file with all env variables and credentials -## Running the server locally +3. Install dependencies: +```bash +pip install -r requirements.txt +``` +4. Create environment variables file: +```bash +touch .env +# Edit .env with your credentials ``` + +5. Start the development server: +```bash python manage.py runserver ``` -## Adding new fields to the data model +The API will be available at `http://localhost:8000/` by default. + +## ๐Ÿ“š API Documentation + +### Interactive Docs + +Once the server is running, access the auto-generated documentation: + +- **Swagger UI**: `http://localhost:8000/docs/` +- **ReDoc**: `http://localhost:8000/redocs/` + +## ๐Ÿ’ป Development + +### Adding New Fields to the Data Model + +1. Update models in `mcserver/models.py`: +```python +class YourModel(models.Model): + new_field = models.CharField(max_length=255) +``` + +2. Create migration: +```bash +python manage.py makemigrations +``` + +3. Apply migration: +```bash +python manage.py migrate # Careful: modifies database! +``` + +4. Update serializers in `mcserver/serializers.py`: +```python +class YourModelSerializer(serializers.ModelSerializer): + class Meta: + fields = [... 'new_field'] +``` + +5. Potentially update `mcserver/admin.py` + +### Running Tests + +```bash +python manage.py test ./tests/ +``` -1. Add fields to `mcserver/models.py` -2. Run `python manage.py makemigrations` -3. Run `python manage.py migrate` (be careful, this modifies the database) -4. Add fields we want to expose in the api to the `mcserver/serializers.py` file +> **Note**: Some tests may be outdated and fail. Test `test_permissions.SessionsPermissionsTests` may fail on Windows but works on Ubuntu and macOS. -Then for deploying to production we pull all the updated code and run the step 3. (with the production `.env` file) +## ๐ŸŒ Internationalization -## Internationalization/Localization +### Adding New Languages -Instructions in this [Link](https://docs.djangoproject.com/en/4.2/topics/i18n/translation/). +Navigate to the `mcserver` folder: -**Note:** You must also install [gettext](https://www.gnu.org/software/gettext/). After install, restart your IDE/Terminal). +1. Create translation files for a language: +```bash +django-admin makemessages -l +# Example: django-admin makemessages -l es +``` + +2. Compile translation messages: +```bash +django-admin compilemessages +``` + +> **Note**: Make sure gettext is installed and your IDE/Terminal is restarted after installation. -Inside of mcserver folder: +## ๐Ÿšข Deployment -1. Create files for a language: +### Production Deployment Steps - `django-admin makemessages -l ` +1. Pull the latest code: +```bash +git pull origin main +``` + +2. Update dependencies: +```bash +pip install -r requirements.txt +``` + +3. Run migrations: +```bash +python manage.py migrate +``` + +4. Restart the application server (Gunicorn/uWSGI/etc.) + +## ๐Ÿงช Testing + +### Test Coverage + +Run tests with coverage: + +```bash +coverage run manage.py test ./tests/ +coverage report -m +``` + +### API Testing + +Use the Swagger UI at `/docs/` or use tools like curl: + +```bash +# Create a session +curl -X POST http://localhost:8000/sessions/ \ + -H "Authorization: Token your_token" \ + -H "Content-Type: application/json" \ + -d '{"subject": "subject_uuid"}' +``` -2. Compile messages: +## ๐Ÿค Contributing - `django-admin compilemessages` +We welcome contributions! Please submit an [Issue](https://github.com/opencap-org/opencap-api/issues) or create a [PR](https://github.com/opencap-org/opencap-api/pulls). +### Development Workflow -## Current routes (not up to date, there are more): +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/AmazingFeature`) +3. Commit your changes (`git commit -m 'Add some AmazingFeature'`) +4. Push to the branch (`git push origin feature/AmazingFeature`) +5. Open a Pull Request -/sessions/new/ -> returns session_id and the QR code -/sessions//status/?device_id= <- devices use this link to register and get video_id +## ๐Ÿ“„ License -/sessions//record/ -> server uses this link to start recording +This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE.md) file for details. -/sessions//stop/ -> server uses this link to stop recording - -/video// <- devices use this link to upload the recorded video and parameters From 25da8c6333aa1236fdd0e7acf8f5ef5042f08fa5 Mon Sep 17 00:00:00 2001 From: Alberto Casas Ortiz Date: Tue, 28 Jul 2026 12:17:02 -0700 Subject: [PATCH 3/5] Reverted S3 changes in models --- mcserver/models.py | 52 +++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/mcserver/models.py b/mcserver/models.py index e85ebbd..e2731e9 100644 --- a/mcserver/models.py +++ b/mcserver/models.py @@ -1,25 +1,24 @@ +import json + +from django.db import models +from django.contrib.auth.models import AbstractUser +from django.core.validators import MinValueValidator, MaxValueValidator import os import uuid +import base64 +import pathlib from http import HTTPStatus - -from django.contrib.auth.models import AbstractUser +from django.utils import timezone from django.core.exceptions import ValidationError -from django.db import models -from django.db.models.signals import post_save, pre_delete +from django.db.models.signals import post_save +from django.contrib.auth.signals import user_logged_in from django.dispatch import receiver -from django.utils import timezone +from rest_framework.authtoken.models import Token from django.utils.translation import gettext as _ +from rest_framework import status +from django.conf import settings -def delete_s3_file(file_field): - """Delete a file from the configured Django storage backend.""" - if not file_field or not file_field.name: - return - - try: - file_field.delete(save=False) - except Exception as e: - print(f"Error deleting file '{file_field.name}': {e}") def random_filename(instance, filename): return "{}-{}".format(uuid.uuid4(), filename) @@ -256,7 +255,8 @@ class ResetPassword(models.Model): datetime = models.DateField(default=timezone.now) from django_otp.plugins.otp_email.models import EmailDevice - +from django.template.loader import render_to_string +from mcserver.customEmailDevice import CustomEmailDevice @receiver(post_save, sender=User) def create_profile(sender, instance, created, **kwargs): @@ -572,25 +572,3 @@ def get_available_data(self, only_public=False, subject_id=None, share_token=Non subject_ids.append(subject.id) return data - -@receiver(pre_delete, sender=Video) -def delete_video_files(sender, instance, **kwargs): - """Delete Video files when the record is deleted.""" - delete_s3_file(instance.video) - delete_s3_file(instance.video_thumb) - delete_s3_file(instance.keypoints) - -@receiver(pre_delete, sender=Result) -def delete_result_files(sender, instance, **kwargs): - """Delete Result media when the record is deleted.""" - delete_s3_file(instance.media) - -@receiver(pre_delete, sender=Session) -def delete_session_files(sender, instance, **kwargs): - """Delete Session QR code when the record is deleted.""" - delete_s3_file(instance.qrcode) - -@receiver(pre_delete, sender=DownloadLog) -def delete_download_log_files(sender, instance, **kwargs): - """Delete DownloadLog archive when the record is deleted.""" - delete_s3_file(instance.media) \ No newline at end of file From 0a45817b3ce15f3bba15f5f05b4ba7a966e1d6ac Mon Sep 17 00:00:00 2001 From: Alberto Casas Ortiz Date: Tue, 28 Jul 2026 12:19:03 -0700 Subject: [PATCH 4/5] Removed coverage, as not integrated yet. --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e1c3a18..d960c37 100644 --- a/README.md +++ b/README.md @@ -165,13 +165,12 @@ python manage.py migrate ## ๐Ÿงช Testing -### Test Coverage +### Test -Run tests with coverage: +Run tests with: ```bash -coverage run manage.py test ./tests/ -coverage report -m +run manage.py test ./tests/ ``` ### API Testing From bcf8e9835cb595f7bd02fd0c36063b4c86ccbca4 Mon Sep 17 00:00:00 2001 From: carmichaelong Date: Fri, 31 Jul 2026 15:54:31 -0700 Subject: [PATCH 5/5] clean up README --- README.md | 182 +++++++++++++++++------------------------------------- 1 file changed, 57 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index d960c37..c837daa 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,14 @@ # ๐ŸŽฏ OpenCap API -[![Python Version](https://img.shields.io/badge/python-3.7+-blue.svg)](https://www.python.org/downloads/) -[![Django Version](https://img.shields.io/badge/django-3.1.14+-green.svg)](https://www.djangoproject.com/) +[![Python Version](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/) +[![Django Version](https://img.shields.io/badge/django-3.1.14-green.svg)](https://www.djangoproject.com/) [![License](https://img.shields.io/badge/license-APACHE_2.0-blue.svg)](LICENSE.md) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/opencap-org/opencap-api/pulls) -> Backend API for OpenCap - The open-source platform for biomechanical motion capture and gait analysis using mobile devices. - -## ๐Ÿ“– Table of Contents - -- [Overview](#overview) -- [Workflow](#workflow) -- [Getting Started](#getting-started) -- [API Documentation](#api-documentation) -- [Development](#development) -- [Testing](#testing) -- [Internationalization](#internationalization) -- [Deployment](#deployment) -- [Contributing](#contributing) -- [License](#license) +> Django backend for [OpenCap](https://app.opencap.ai). ## ๐Ÿ”ญ Overview -OpenCap is an open source biomechanical motion capture platform that leverages iOS devices to capture and analyze human movement. This repository contains the Django-based backend API that orchestrates the entire workflow, from session management to video processing and biomechanical analysis. +The API serves both the webapp and the iOS app. It stores sessions, subjects, trials, and videos, and queues recordings for the processing pipeline. Background work (archive builds, session downloads, scheduled cleanup) runs in Celery workers backed by Redis, not in the web process. ## ๐Ÿ”„ Workflow @@ -42,163 +28,109 @@ OpenCap is an open source biomechanical motion capture platform that leverages i ### Prerequisites -- Python 3.7+ -- [gettext](https://www.gnu.org/software/gettext/) (for internationalization) +Beyond `requirements.txt`, which covers pip packages only: + +- **Python 3.7** โ€” the pinned dependencies do not install on newer versions +- **PostgreSQL** โ€” `mcserver/settings.py` always uses the `postgresql` backend + +Depending on the work: + +- **Redis** โ€” to run Celery. The API serves requests without it, but asynchronous work (session downloads, cleanup jobs) never runs +- **[gettext](https://www.gnu.org/software/gettext/)** โ€” to compile translations ### Installation -1. Clone the repository: ```bash git clone https://github.com/opencap-org/opencap-api.git cd opencap-api -``` - -2. Create and activate a conda environment: -```bash conda create -n opencap python=3.7 conda activate opencap -``` - -3. Install dependencies: -```bash pip install -r requirements.txt ``` -4. Create environment variables file: -```bash -touch .env -# Edit .env with your credentials -``` +### Configuration + +Create a `.env` file in the repository root. The required variables are the `config(...)` calls in `mcserver/settings.py` that have no default; ask a maintainer for development values. + +### Running locally -5. Start the development server: ```bash +python manage.py migrate python manage.py runserver ``` -The API will be available at `http://localhost:8000/` by default. +The API is served at `http://localhost:8000/`. -## ๐Ÿ“š API Documentation +Asynchronous work needs Celery processes running alongside the server: -### Interactive Docs +```bash +celery -A mcserver worker -l info # session downloads, archive builds +celery -A mcserver beat -l info # scheduled jobs +``` -Once the server is running, access the auto-generated documentation: +## ๐Ÿ“š API Documentation + +With the server running: - **Swagger UI**: `http://localhost:8000/docs/` - **ReDoc**: `http://localhost:8000/redocs/` -## ๐Ÿ’ป Development - -### Adding New Fields to the Data Model - -1. Update models in `mcserver/models.py`: -```python -class YourModel(models.Model): - new_field = models.CharField(max_length=255) -``` +Routes are registered in `mcserver/urls.py`. Most come from the viewsets in `mcserver/views.py`, which also add a number of custom actions on top of the standard REST routes. Requests authenticate with a DRF token, or a session cookie for the browsable API. -2. Create migration: -```bash -python manage.py makemigrations -``` +## ๐Ÿ’ป Development -3. Apply migration: -```bash -python manage.py migrate # Careful: modifies database! -``` +### Adding new fields to the data model -4. Update serializers in `mcserver/serializers.py`: -```python -class YourModelSerializer(serializers.ModelSerializer): - class Meta: - fields = [... 'new_field'] -``` +1. Add the field to the model in `mcserver/models.py` +2. Run `python manage.py makemigrations` +3. Run `python manage.py migrate` โ€” careful, this modifies the database +4. Add the field to `mcserver/serializers.py` if the API should expose it +5. Update `mcserver/admin.py` if it should appear in the admin -5. Potentially update `mcserver/admin.py` +### Running tests -### Running Tests +Tests need a valid `.env` and a database they are allowed to create. ```bash -python manage.py test ./tests/ +python manage.py test tests # whole suite +python manage.py test tests.test_permissions # one module ``` > **Note**: Some tests may be outdated and fail. Test `test_permissions.SessionsPermissionsTests` may fail on Windows but works on Ubuntu and macOS. ## ๐ŸŒ Internationalization -### Adding New Languages +See the [Django translation docs](https://docs.djangoproject.com/en/3.1/topics/i18n/translation/). This requires [gettext](https://www.gnu.org/software/gettext/) โ€” restart your terminal or IDE after installing it. -Navigate to the `mcserver` folder: - -1. Create translation files for a language: -```bash -django-admin makemessages -l -# Example: django-admin makemessages -l es -``` +From the `mcserver` folder: -2. Compile translation messages: ```bash -django-admin compilemessages +django-admin makemessages -l es # create or refresh files for a language +django-admin compilemessages # compile them ``` -> **Note**: Make sure gettext is installed and your IDE/Terminal is restarted after installation. - ## ๐Ÿšข Deployment -### Production Deployment Steps +Deployment is automated with GitHub Actions. Each push builds `Dockerfile`, pushes the image to ECR, and forces a new ECS deployment. -1. Pull the latest code: -```bash -git pull origin main -``` +| Branch | Workflow | Effect | +| --- | --- | --- | +| `dev` | `.github/workflows/ecr-dev.yml` | Builds `opencap/api-dev`; redeploys `api-server-dev`, `api-server-celery-dev`, `api-server-celery-beat-dev` in `opencap-api-cluster-dev` | +| `main` | `.github/workflows/ecr.yml` | Builds `opencap/api`; redeploys `api-server`, `api-server-celery`, `api-server-celery-beat` in `opencap-api-cluster` | -2. Update dependencies: -```bash -pip install -r requirements.txt -``` - -3. Run migrations: -```bash -python manage.py migrate -``` +Two things are not automated: -4. Restart the application server (Gunicorn/uWSGI/etc.) - -## ๐Ÿงช Testing - -### Test - -Run tests with: - -```bash -run manage.py test ./tests/ -``` - -### API Testing - -Use the Swagger UI at `/docs/` or use tools like curl: - -```bash -# Create a session -curl -X POST http://localhost:8000/sessions/ \ - -H "Authorization: Token your_token" \ - -H "Content-Type: application/json" \ - -d '{"subject": "subject_uuid"}' -``` +- **Migrations.** If your change adds one, run `python manage.py migrate` against that environment's database after the deploy. +- **Environment variables.** New settings have to be added to the ECS task definitions; they are not read from this repository. ## ๐Ÿค Contributing -We welcome contributions! Please submit an [Issue](https://github.com/opencap-org/opencap-api/issues) or create a [PR](https://github.com/opencap-org/opencap-api/pulls). - -### Development Workflow - -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/AmazingFeature`) -3. Commit your changes (`git commit -m 'Add some AmazingFeature'`) -4. Push to the branch (`git push origin feature/AmazingFeature`) -5. Open a Pull Request +1. Open an [issue](https://github.com/opencap-org/opencap-api/issues) describing the change first +2. Branch off `dev` +3. Open a pull request against `dev`, referencing the issue +`dev` is the integration branch. Changes reach production through a `dev` โ†’ `main` pull request. Since a push to `main` deploys production immediately, work should not target it directly. ## ๐Ÿ“„ License -This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE.md) file for details. - +Apache License 2.0 โ€” see [LICENSE.md](LICENSE.md) for details.