From f0cf3af5117e70dbe7fdf739996dbed0eba50a43 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Sun, 20 Sep 2026 15:42:11 +0530 Subject: [PATCH 1/4] feat(api): link-only dataset import from Hugging Face, GitHub and Kaggle Let a publisher add a dataset that already lives on a third-party platform by giving its identifier (or page URL). Only metadata is fetched; files are never copied or listed, and downloads redirect to the platform. - previewPlatformDataset query: fetch title, description, license, tags, author and last-updated from the platform, with no side effects. - importPlatformDataset mutation: create a DRAFT dataset prefilled from the platform, attach sectors/geographies whose names match the platform's tags, fill matching dataset metadata fields, add one EXTERNAL resource linking to the dataset page, record provenance, and grant the owner role, all in one transaction. Optional title override. Duplicate imports within the same organisation/user are rejected. - DatasetSource model (one-to-one with Dataset) for provenance; exposed as TypeDataset.source. TypeResource.url is now exposed. - Importers for Hugging Face, GitHub and Kaggle behind a registry; all work without API keys for public datasets. HF_TOKEN, GITHUB_TOKEN and KAGGLE_USERNAME/KAGGLE_KEY are optional. - Download view redirects EXTERNAL resources to their URL and returns 404 instead of raising when a resource has no file. - Dataset search document gains source_platform; /api/search/dataset/ returns it, aggregates on it and filters by it (NATIVE = not imported). formats indexing skips resources without file details. After deploy, run `manage.py search_index --rebuild` once so Elasticsearch maps source_platform as a keyword before the first import is indexed. Refs CivicDataLab/DataSpace#174, #190, #191, #192 --- .env.example | 6 + DataSpace/settings.py | 10 + api/migrations/0048_platform_import.py | 75 ++++++ api/models/Dataset.py | 26 +- api/models/DatasetSource.py | 46 ++++ api/models/__init__.py | 1 + api/schema/platform_import_schema.py | 99 +++++++ api/schema/schema.py | 3 + api/services/platform_import_service.py | 241 ++++++++++++++++++ api/services/platform_importers/__init__.py | 64 +++++ api/services/platform_importers/base.py | 196 ++++++++++++++ api/services/platform_importers/github.py | 144 +++++++++++ .../platform_importers/huggingface.py | 165 ++++++++++++ api/services/platform_importers/kaggle.py | 121 +++++++++ api/types/type_dataset.py | 7 + api/types/type_dataset_source.py | 62 +++++ api/types/type_resource.py | 1 + api/utils/enums.py | 8 + api/views/download_view.py | 43 ++-- api/views/search_dataset.py | 10 + api/views/search_unified.py | 1 + search/documents/dataset_document.py | 8 + 22 files changed, 1310 insertions(+), 27 deletions(-) create mode 100644 api/migrations/0048_platform_import.py create mode 100644 api/models/DatasetSource.py create mode 100644 api/schema/platform_import_schema.py create mode 100644 api/services/platform_import_service.py create mode 100644 api/services/platform_importers/__init__.py create mode 100644 api/services/platform_importers/base.py create mode 100644 api/services/platform_importers/github.py create mode 100644 api/services/platform_importers/huggingface.py create mode 100644 api/services/platform_importers/kaggle.py create mode 100644 api/types/type_dataset_source.py diff --git a/.env.example b/.env.example index 58fefe7e..f1a77c70 100644 --- a/.env.example +++ b/.env.example @@ -12,3 +12,9 @@ URL_WHITELIST=http://localhost:8000,http://localhost,http://localhost:3000 DEBUG=True SECRET_KEY=your-secret-key REDIS_URL=redis://redis:6379/1 + +# Third-party platform imports (optional) +KAGGLE_USERNAME= +KAGGLE_KEY= +HF_TOKEN= +GITHUB_TOKEN= diff --git a/DataSpace/settings.py b/DataSpace/settings.py index 4d25acca..f3fcd32c 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -289,6 +289,16 @@ } +# Third-party platform imports (link-only). All three platforms work with no +# key for public datasets. KAGGLE_* adds a file count to Kaggle imports, +# HF_TOKEN unlocks gated Hugging Face repos, GITHUB_TOKEN lifts GitHub's +# anonymous rate limit. +KAGGLE_USERNAME = os.getenv("KAGGLE_USERNAME", None) +KAGGLE_KEY = os.getenv("KAGGLE_KEY", None) +HF_TOKEN = os.getenv("HF_TOKEN", None) +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", None) # optional, lifts the 60 req/hour anonymous limit +PLATFORM_IMPORT_TIMEOUT = float(os.getenv("PLATFORM_IMPORT_TIMEOUT", "15")) + # DVC settings DVC_REPO_PATH = os.path.join(BASE_DIR, "dvc") DVC_REMOTE_NAME = os.getenv("DVC_REMOTE_NAME", None) diff --git a/api/migrations/0048_platform_import.py b/api/migrations/0048_platform_import.py new file mode 100644 index 00000000..ac2e5107 --- /dev/null +++ b/api/migrations/0048_platform_import.py @@ -0,0 +1,75 @@ +# Generated by Django 5.0.4 on 2026-09-20 10:13 + +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0047_resourcetype_publication_collaborative_publications_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="DatasetSource", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ( + "platform", + models.CharField( + choices=[ + ("KAGGLE", "Kaggle"), + ("HUGGINGFACE", "Huggingface"), + ("GITHUB", "Github"), + ], + max_length=50, + ), + ), + ("source_identifier", models.CharField(max_length=300)), + ("source_url", models.URLField(max_length=500)), + ("source_author", models.CharField(blank=True, max_length=300)), + ("source_license", models.CharField(blank=True, max_length=300)), + ("source_last_updated", models.DateTimeField(blank=True, null=True)), + ("raw_metadata", models.JSONField(blank=True, default=dict)), + ("imported_at", models.DateTimeField(auto_now_add=True)), + ("last_synced_at", models.DateTimeField(auto_now=True)), + ( + "dataset", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="source", + to="api.dataset", + ), + ), + ( + "imported_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="imported_dataset_sources", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "db_table": "dataset_source", + "indexes": [ + models.Index( + fields=["platform", "source_identifier"], + name="dataset_sou_platfor_38ca23_idx", + ) + ], + }, + ), + ] diff --git a/api/models/Dataset.py b/api/models/Dataset.py index a2b99a1a..296f509c 100644 --- a/api/models/Dataset.py +++ b/api/models/Dataset.py @@ -1,5 +1,5 @@ import uuid -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional from django.db import models from django.db.models import Sum @@ -124,14 +124,22 @@ def formats_indexing(self) -> list[str]: Used in Elasticsearch indexing. """ - return list( - set( - [ - resource.resourcefiledetails.format # type: ignore - for resource in self.resources.all() - ] - ).difference({""}) - ) + formats: set[str] = set() + for resource in self.resources.all(): + # Link-only (EXTERNAL) resources have no file details; skip them. + file_details = getattr(resource, "resourcefiledetails", None) + if file_details is not None and file_details.format: + formats.add(file_details.format) + return list(formats) + + @property + def source_platform_indexing(self) -> Optional[str]: + """Platform this dataset was imported from, or None for native datasets. + + Used in Elasticsearch indexing. + """ + source = getattr(self, "source", None) + return source.platform if source is not None else None @property def catalogs_indexing(self) -> list[str]: diff --git a/api/models/DatasetSource.py b/api/models/DatasetSource.py new file mode 100644 index 00000000..e9aa7817 --- /dev/null +++ b/api/models/DatasetSource.py @@ -0,0 +1,46 @@ +import uuid + +from django.db import models + +from api.utils.enums import ImportPlatform + + +class DatasetSource(models.Model): + """Provenance record for a dataset imported from a third-party platform. + + Imports are link-only: DataSpace never copies the platform's files. This + row remembers where the dataset came from so the UI can attribute it, link + back to it, and (later) re-sync its metadata. The raw platform response is + kept in ``raw_metadata`` for debugging and future field mapping. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + dataset = models.OneToOneField("api.Dataset", on_delete=models.CASCADE, related_name="source") + platform = models.CharField(max_length=50, choices=ImportPlatform.choices) + # Platform-native identifier, e.g. "owner/dataset-slug" (Kaggle) or + # "namespace/name" (Hugging Face). Normalised by the importer. + source_identifier = models.CharField(max_length=300) + # Human-facing page on the platform. + source_url = models.URLField(max_length=500) + source_author = models.CharField(max_length=300, blank=True) + # License string exactly as the platform reported it (may not map onto + # DatasetLicense; the mapped value lives on Dataset.license). + source_license = models.CharField(max_length=300, blank=True) + source_last_updated = models.DateTimeField(null=True, blank=True) + raw_metadata = models.JSONField(default=dict, blank=True) + imported_by = models.ForeignKey( + "authorization.User", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="imported_dataset_sources", + ) + imported_at = models.DateTimeField(auto_now_add=True) + last_synced_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = "dataset_source" + indexes = [models.Index(fields=["platform", "source_identifier"])] + + def __str__(self) -> str: + return f"{self.platform}:{self.source_identifier}" diff --git a/api/models/__init__.py b/api/models/__init__.py index 6c54b34c..19f7c515 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -9,6 +9,7 @@ ) from api.models.Dataset import Dataset, Tag from api.models.DatasetMetadata import DatasetMetadata +from api.models.DatasetSource import DatasetSource from api.models.DataSpace import DataSpace from api.models.Geography import Geography from api.models.Metadata import Metadata diff --git a/api/schema/platform_import_schema.py b/api/schema/platform_import_schema.py new file mode 100644 index 00000000..12de66ad --- /dev/null +++ b/api/schema/platform_import_schema.py @@ -0,0 +1,99 @@ +"""GraphQL surface for link-only imports from third-party platforms. + +- ``preview_platform_dataset`` fetches normalised metadata, no side effects. +- ``import_platform_dataset`` creates a DRAFT dataset with one EXTERNAL resource + linking to the dataset page on the platform (files are not imported). + +Both take the same organization/dataspace request headers as ``add_dataset``; +the imported dataset is owned the same way a manually created one would be. +""" + +from typing import Optional + +import strawberry +from strawberry.types import Info + +from api.schema.base_mutation import ( + BaseMutation, + GraphQLValidationError, + MutationResponse, +) +from api.services.platform_import_service import ( + import_platform_dataset, + preview_platform_dataset, +) +from api.services.platform_importers import PlatformImportError +from api.types.type_dataset import TypeDataset +from api.types.type_dataset_source import ( + TypePlatformDatasetPreview, + import_platform_enum, +) +from api.utils.graphql_telemetry import trace_resolver +from authorization.graphql_permissions import IsAuthenticated +from authorization.permissions import CreateDatasetPermission + + +@strawberry.input +class ImportPlatformDatasetInput: + platform: import_platform_enum # type: ignore + #: Short id ("owner/name") or a pasted platform URL. + identifier: str + #: Optional display title on DataSpace; defaults to the platform's title. + title: Optional[str] = None + + +@strawberry.type +class Query: + @strawberry.field(permission_classes=[IsAuthenticated]) + @trace_resolver(name="preview_platform_dataset", attributes={"component": "platform_import"}) + def preview_platform_dataset( + self, info: Info, platform: import_platform_enum, identifier: str # type: ignore + ) -> TypePlatformDatasetPreview: + """Look up a Hugging Face / GitHub / Kaggle dataset and show what an import would create.""" + try: + data = preview_platform_dataset(platform.value, identifier) + except PlatformImportError as exc: + # Surface the importer's user-safe message as a GraphQL error. + raise ValueError(exc.message) from exc + return TypePlatformDatasetPreview.from_info(data) + + +@strawberry.type +class Mutation: + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[IsAuthenticated, CreateDatasetPermission], + trace_name="import_platform_dataset", + trace_attributes={"component": "platform_import"}, + track_activity={ + "verb": "imported", + "get_data": lambda result, import_input=None, **kwargs: { + "dataset_id": str(result.id), + "dataset_title": result.title, + "platform": import_input.platform.value if import_input else None, + "identifier": import_input.identifier if import_input else None, + "organization": (str(result.organization.id) if result.organization else None), + }, + }, + ) + def import_platform_dataset( + self, info: Info, import_input: ImportPlatformDatasetInput + ) -> MutationResponse[TypeDataset]: + """Create a DRAFT dataset that links to the dataset on the platform.""" + organization = info.context.context.get("organization") + dataspace = info.context.context.get("dataspace") + user = info.context.user + + try: + dataset = import_platform_dataset( + platform=import_input.platform.value, + identifier=import_input.identifier, + user=user, + organization=organization, + dataspace=dataspace, + title=import_input.title, + ) + except PlatformImportError as exc: + return MutationResponse.error_response(GraphQLValidationError.from_message(exc.message)) + + return MutationResponse.success_response(TypeDataset.from_django(dataset)) diff --git a/api/schema/schema.py b/api/schema/schema.py index 4678b63d..5f919cfc 100644 --- a/api/schema/schema.py +++ b/api/schema/schema.py @@ -16,6 +16,7 @@ import api.schema.metadata_schema import api.schema.organization_data_schema import api.schema.organization_schema +import api.schema.platform_import_schema import api.schema.publication_schema import api.schema.resource_chart_schema import api.schema.resource_schema @@ -77,6 +78,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.user_schema.Query, api.schema.collaborative_schema.Query, api.schema.publication_schema.Query, + api.schema.platform_import_schema.Query, AuthQuery, ), ) @@ -100,6 +102,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.tags_schema.Mutation, api.schema.collaborative_schema.Mutation, api.schema.publication_schema.Mutation, + api.schema.platform_import_schema.Mutation, AuthMutation, ), ) diff --git a/api/services/platform_import_service.py b/api/services/platform_import_service.py new file mode 100644 index 00000000..b30fe17b --- /dev/null +++ b/api/services/platform_import_service.py @@ -0,0 +1,241 @@ +"""Turn a third-party platform dataset into a DataSpace Dataset (link-only). + +Creates the Dataset (DRAFT), a single EXTERNAL Resource that links to the +dataset's page on the platform, a DatasetSource provenance row, tags, taxonomy +and metadata prefill, and the creator's owner permission — all inside one +transaction. Files are never listed or copied: people reach them through the +original dataset link. +""" + +from __future__ import annotations + +from typing import Iterable, Optional + +import structlog +from django.db import transaction +from django.utils.text import slugify + +from api.models import ( + Dataset, + DatasetMetadata, + DatasetSource, + DataSpace, + Geography, + Metadata, + Organization, + Resource, + Sector, + Tag, +) +from api.services.platform_importers import ( + PlatformDatasetInfo, + PlatformImportError, + get_importer, +) +from api.utils.enums import ( + DatasetAccessType, + DatasetStatus, + DatasetType, + DataType, + MetadataModels, +) +from authorization.models import DatasetPermission, Role, User + +logger = structlog.get_logger("dataspace.platform_import") + + +class DuplicateImportError(PlatformImportError): + """The same platform dataset was already imported into this publisher scope.""" + + def __init__(self, existing: Dataset) -> None: + super().__init__( + f"This dataset was already imported as '{existing.title}' ({existing.slug})" + ) + self.existing = existing + + +def preview_platform_dataset(platform: str, identifier: str) -> PlatformDatasetInfo: + """Fetch normalised metadata without creating anything.""" + importer = get_importer(platform) + return importer.fetch_dataset_info(identifier) + + +def find_existing_import( + platform: str, + identifier: str, + organization: Optional[Organization], + user: Optional[User], +) -> Optional[Dataset]: + """Return a dataset already imported from this source in the same scope. + + Scope is the organization when importing on its behalf, otherwise the + individual user — mirroring how datasets are owned elsewhere. + """ + qs = DatasetSource.objects.select_related("dataset").filter( + platform=platform, source_identifier=identifier + ) + if organization is not None: + qs = qs.filter(dataset__organization=organization) + else: + qs = qs.filter(dataset__organization__isnull=True, dataset__user=user) + source = qs.first() + return source.dataset if source else None + + +def _unique_dataset_slug(title: str) -> str: + """Pick a free slug up front. Dataset.save() retries on a slug collision, + but only a handful of times; platform titles ("Iris", "Wine Reviews") + repeat across organisations far more often than timestamped native ones, + so choose a free slug before saving rather than rely on the retry.""" + base = slugify(title)[:240] or "imported-dataset" + slug, counter = base, 2 + while Dataset.objects.filter(slug=slug).exists(): + slug = f"{base}-{counter}" + counter += 1 + return slug + + +PLATFORM_LABELS = {"HUGGINGFACE": "Hugging Face", "KAGGLE": "Kaggle", "GITHUB": "GitHub"} + + +# Platform tags/topics are matched (case-insensitively) against our own +# taxonomies so the publisher lands on the metadata step with sectors and +# geographies already ticked where the names line up. Publishing requires +# sectors, so this is the most valuable prefill we can do without a human. +def _prefill_taxonomies(dataset: Dataset, tags: Iterable[str]) -> None: + names = {t.strip().lower() for t in tags if t and t.strip()} + if not names: + return + sectors = [s for s in Sector.objects.all() if s.name.strip().lower() in names] + if sectors: + dataset.sectors.add(*sectors) + geographies = [g for g in Geography.objects.all() if g.name.strip().lower() in names] + if geographies: + dataset.geographies.add(*geographies) + + +# Optional EAV prefill: if the deployment defines dataset metadata fields whose +# label matches one of these (case-insensitive), fill it from the platform. +# Deployments without such fields are simply skipped. +METADATA_LABEL_SOURCES = { + "source": "source_url", + "source url": "source_url", + "source platform": "platform_label", + "original source": "source_url", + "author": "author", + "creator": "author", + "publisher": "author", + "license": "license", + "original license": "license", + "last updated": "last_updated", + "source last updated": "last_updated", +} + + +def _prefill_metadata(dataset: Dataset, info: PlatformDatasetInfo) -> None: + values = { + "source_url": info.source_url, + "platform_label": PLATFORM_LABELS.get(str(info.platform), str(info.platform).title()), + "author": info.author, + "license": info.license, + "last_updated": info.last_updated.date().isoformat() if info.last_updated else "", + } + fields = Metadata.objects.filter(enabled=True, model=MetadataModels.DATASET) + for field in fields: + source_key = METADATA_LABEL_SOURCES.get((field.label or "").strip().lower()) + value = values.get(source_key or "", "") + if not value: + continue + try: + DatasetMetadata(dataset=dataset, metadata_item=field, value=str(value)[:1000]).save() + except Exception as exc: # validators on the field may reject the value; that's fine + logger.info("platform_import_metadata_skipped", label=field.label, error=str(exc)) + + +@transaction.atomic +def import_platform_dataset( + *, + platform: str, + identifier: str, + user: User, + organization: Optional[Organization] = None, + dataspace: Optional[DataSpace] = None, + info: Optional[PlatformDatasetInfo] = None, + title: Optional[str] = None, +) -> Dataset: + """Create a DRAFT dataset from a platform source. Raises PlatformImportError.""" + importer = get_importer(platform) + canonical_id = importer.parse_identifier(identifier) + + existing = find_existing_import(platform, canonical_id, organization, user) + if existing is not None: + raise DuplicateImportError(existing) + + if info is None: + info = importer.fetch_dataset_info(canonical_id) + + # Publisher may choose the name shown on DataSpace; platform title otherwise. + display_title = (title or "").strip()[:300] or info.title + dataset = Dataset.objects.create( + title=display_title, + slug=_unique_dataset_slug(display_title), + description=info.description, + user=user, + organization=organization, + dataspace=dataspace, + status=DatasetStatus.DRAFT, + access_type=DatasetAccessType.PUBLIC, + license=info.mapped_license, + dataset_type=DatasetType.DATA, + ) + + if info.tags: + tags = [ + Tag.objects.get_or_create(defaults={"value": value}, value__iexact=value)[0] + for value in info.tags + ] + dataset.tags.set(tags) + _prefill_taxonomies(dataset, info.tags) + _prefill_metadata(dataset, info) + + # One resource standing for the whole dataset on the platform. Its URL is + # the dataset page, so "download" redirects there and people browse/fetch + # files with the platform's own tooling. + platform_label = PLATFORM_LABELS.get(str(info.platform), str(info.platform).title()) + Resource.objects.create( + dataset=dataset, + type=DataType.EXTERNAL, + name=f"Dataset on {platform_label}"[:200], + url=info.source_url[:500], + description=f"Files are hosted on {platform_label}. Open the link to browse and download them.", + ) + + DatasetSource.objects.create( + dataset=dataset, + platform=info.platform, + source_identifier=info.identifier, + source_url=info.source_url[:500], + source_author=info.author[:300], + source_license=info.license[:300], + source_last_updated=info.last_updated, + raw_metadata=info.raw, + imported_by=user, + ) + + try: + owner_role = Role.objects.get(name="owner") + except Role.DoesNotExist as exc: + # Same seed the rest of the app relies on (add_dataset does a bare .get()). + raise PlatformImportError( + "Roles are not initialised on this server (run `manage.py init_roles`)" + ) from exc + DatasetPermission.objects.create(user=user, dataset=dataset, role=owner_role) + + logger.info( + "platform_dataset_imported", + platform=info.platform, + identifier=info.identifier, + dataset_id=str(dataset.id), + user_id=str(user.id), + ) + return dataset diff --git a/api/services/platform_importers/__init__.py b/api/services/platform_importers/__init__.py new file mode 100644 index 00000000..246c6b08 --- /dev/null +++ b/api/services/platform_importers/__init__.py @@ -0,0 +1,64 @@ +"""Registry of third-party platform importers. + +Add a platform by subclassing ``PlatformImporter`` and registering it here; +the GraphQL layer and import service never reference a concrete importer. +""" + +from typing import Dict, Optional, Type +from urllib.parse import urlparse + +from api.services.platform_importers.base import ( + InvalidIdentifierError, + PlatformAuthError, + PlatformDatasetInfo, + PlatformDatasetNotFoundError, + PlatformImporter, + PlatformImportError, + PlatformUnavailableError, +) +from api.services.platform_importers.github import GitHubImporter +from api.services.platform_importers.huggingface import HuggingFaceImporter +from api.services.platform_importers.kaggle import KaggleImporter +from api.utils.enums import ImportPlatform + +IMPORTERS: Dict[str, Type[PlatformImporter]] = { + ImportPlatform.KAGGLE: KaggleImporter, + ImportPlatform.HUGGINGFACE: HuggingFaceImporter, + ImportPlatform.GITHUB: GitHubImporter, +} + + +def get_importer(platform: str) -> PlatformImporter: + try: + return IMPORTERS[str(platform)]() + except KeyError as exc: + raise InvalidIdentifierError(f"Unsupported platform: {platform}") from exc + + +def detect_platform(value: str) -> Optional[str]: + """Guess the platform from a pasted URL (None for bare identifiers).""" + raw = (value or "").strip() + if not raw: + return None + host = urlparse(raw if "://" in raw else f"https://{raw}").netloc.lower() + for platform, importer in IMPORTERS.items(): + if host in importer.hosts: + return platform + return None + + +__all__ = [ + "IMPORTERS", + "get_importer", + "detect_platform", + "PlatformImporter", + "PlatformDatasetInfo", + "PlatformImportError", + "InvalidIdentifierError", + "PlatformDatasetNotFoundError", + "PlatformAuthError", + "PlatformUnavailableError", + "HuggingFaceImporter", + "GitHubImporter", + "KaggleImporter", +] diff --git a/api/services/platform_importers/base.py b/api/services/platform_importers/base.py new file mode 100644 index 00000000..bc6a671a --- /dev/null +++ b/api/services/platform_importers/base.py @@ -0,0 +1,196 @@ +"""Shared contract for third-party platform importers. + +An importer turns a user-supplied identifier (short id or pasted URL) into a +normalised ``PlatformDatasetInfo`` by calling the platform's public API. +Importers fetch *metadata only* — title, description, license, tags, author, +last updated and the dataset's page URL. Files are never listed or copied. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, List, Optional + +import requests +import structlog +from django.conf import settings +from django.utils.dateparse import parse_datetime + +from api.utils.enums import DatasetLicense, ImportPlatform + +logger = structlog.get_logger("dataspace.platform_import") + + +# --------------------------------------------------------------------------- # +# Errors +# --------------------------------------------------------------------------- # +class PlatformImportError(Exception): + """Base class for importer failures. ``message`` is safe to show to users.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class InvalidIdentifierError(PlatformImportError): + """The identifier/URL does not look like anything the platform accepts.""" + + +class PlatformDatasetNotFoundError(PlatformImportError): + """The platform reported no dataset for this identifier.""" + + +class PlatformAuthError(PlatformImportError): + """Credentials are missing/invalid, or the dataset is private/gated.""" + + +class PlatformUnavailableError(PlatformImportError): + """Network failure, timeout, rate limit, or a 5xx from the platform.""" + + +# --------------------------------------------------------------------------- # +# Normalised result +# --------------------------------------------------------------------------- # +@dataclass +class PlatformDatasetInfo: + platform: str + identifier: str + title: str + description: str + source_url: str + author: str = "" + license: str = "" + tags: List[str] = field(default_factory=list) + last_updated: Optional[datetime] = None + raw: Dict[str, Any] = field(default_factory=dict) + + @property + def mapped_license(self) -> str: + return map_license(self.license) + + +# --------------------------------------------------------------------------- # +# Helpers shared by importers +# --------------------------------------------------------------------------- # +# Platform license strings (lower-cased) -> DatasetLicense. Anything not listed +# falls back to CC-BY 4.0 and the raw string is kept on DatasetSource. +LICENSE_ALIASES: Dict[str, str] = { + "cc-by-4.0": DatasetLicense.CC_BY_4_0_ATTRIBUTION, + "cc-by": DatasetLicense.CC_BY_4_0_ATTRIBUTION, + "cc by 4.0": DatasetLicense.CC_BY_4_0_ATTRIBUTION, + "attribution 4.0 international (cc by 4.0)": DatasetLicense.CC_BY_4_0_ATTRIBUTION, + "cc-by-sa-4.0": DatasetLicense.CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE, + "cc-by-sa": DatasetLicense.CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE, + "attribution-sharealike 4.0 international (cc by-sa 4.0)": ( + DatasetLicense.CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE + ), + "odc-by": DatasetLicense.OPEN_DATA_COMMONS_BY_ATTRIBUTION, + "odc-by-1.0": DatasetLicense.OPEN_DATA_COMMONS_BY_ATTRIBUTION, + "odc attribution license (odc-by)": DatasetLicense.OPEN_DATA_COMMONS_BY_ATTRIBUTION, + "odbl": DatasetLicense.OPEN_DATABASE_LICENSE, + "odbl-1.0": DatasetLicense.OPEN_DATABASE_LICENSE, + "odc-odbl": DatasetLicense.OPEN_DATABASE_LICENSE, + "database: open database, contents: database contents": DatasetLicense.OPEN_DATABASE_LICENSE, + "database: open database, contents: © original authors": DatasetLicense.OPEN_DATABASE_LICENSE, +} + + +def map_license(raw: str) -> str: + """Map a platform license string onto DatasetLicense (default CC-BY-4.0).""" + key = (raw or "").strip().lower() + return LICENSE_ALIASES.get(key, DatasetLicense.CC_BY_4_0_ATTRIBUTION) + + +def parse_iso_datetime(value: Optional[str]) -> Optional[datetime]: + """Parse a platform timestamp. Django's parser copes with a trailing ``Z`` + and any number of fractional digits (Kaggle sends e.g. ``...:04.7Z``), + which ``datetime.fromisoformat`` on Python 3.10 does not.""" + if not value: + return None + try: + return parse_datetime(value) + except ValueError: + return None + + +# --------------------------------------------------------------------------- # +# Base importer +# --------------------------------------------------------------------------- # +class PlatformImporter(ABC): + """Contract every platform importer implements.""" + + platform: str + #: Human name used in user-facing messages, e.g. "Hugging Face". + label: str = "" + #: Hostnames whose URLs this importer can parse (used by parse_identifier). + hosts: tuple = () + + def __init__(self, session: Optional[requests.Session] = None) -> None: + self.session = session or requests.Session() + self.timeout: float = float(getattr(settings, "PLATFORM_IMPORT_TIMEOUT", 15)) + + @abstractmethod + def parse_identifier(self, value: str) -> str: + """Normalise a short id or pasted URL to the platform's canonical id. + + Raises InvalidIdentifierError when it cannot. + """ + + @abstractmethod + def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: + """Call the platform API and return normalised metadata.""" + + # -- HTTP plumbing ------------------------------------------------------- # + def _get(self, url: str, **kwargs: Any) -> requests.Response: + """GET with uniform error mapping. Subclasses add auth via kwargs.""" + kwargs.setdefault("timeout", self.timeout) + name = self.label or str(self.platform) + try: + response = self.session.get(url, **kwargs) + except requests.Timeout as exc: + raise PlatformUnavailableError(f"{name} timed out") from exc + except requests.RequestException as exc: + logger.warning( + "platform_import_request_failed", platform=self.platform, url=url, error=str(exc) + ) + raise PlatformUnavailableError(f"Could not reach {name}") from exc + + if response.status_code == 404: + raise PlatformDatasetNotFoundError(f"Dataset not found on {name}") + if response.status_code in (401, 403): + raise PlatformAuthError( + f"{name} refused access. The dataset may be private/gated, " + "or the server credentials are missing or invalid." + ) + if response.status_code == 429: + raise PlatformUnavailableError(f"{name} rate limit reached, try again later") + if response.status_code >= 500: + raise PlatformUnavailableError(f"{name} returned an error ({response.status_code})") + if response.status_code >= 400: + raise PlatformImportError(f"{name} rejected the request ({response.status_code})") + return response + + def _get_json(self, url: str, **kwargs: Any) -> Any: + response = self._get(url, **kwargs) + try: + return response.json() + except ValueError as exc: + raise PlatformUnavailableError( + f"{self.label or self.platform} returned an unreadable response" + ) from exc + + +__all__ = [ + "ImportPlatform", + "PlatformImporter", + "PlatformDatasetInfo", + "PlatformImportError", + "InvalidIdentifierError", + "PlatformDatasetNotFoundError", + "PlatformAuthError", + "PlatformUnavailableError", + "map_license", + "parse_iso_datetime", +] diff --git a/api/services/platform_importers/github.py b/api/services/platform_importers/github.py new file mode 100644 index 00000000..c5202e31 --- /dev/null +++ b/api/services/platform_importers/github.py @@ -0,0 +1,144 @@ +"""GitHub repository importer (metadata only). + +A repo (optionally a branch and sub-folder) is treated as a dataset. Two +requests per import: the repo metadata and the README. Public repos need no +token; ``GITHUB_TOKEN`` (settings) is sent when present, mainly to lift the +anonymous 60 requests/hour rate limit. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Optional, Tuple +from urllib.parse import urlparse + +from django.conf import settings + +from api.services.platform_importers.base import ( + InvalidIdentifierError, + PlatformDatasetInfo, + PlatformImporter, + PlatformImportError, + parse_iso_datetime, +) +from api.utils.enums import ImportPlatform + +GITHUB_HOSTS = ("github.com", "www.github.com") +GITHUB_API = "https://api.github.com" +GITHUB_WEB = "https://github.com" + +_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") +MAX_DESCRIPTION = 1000 + + +class GitHubImporter(PlatformImporter): + platform = ImportPlatform.GITHUB + label = "GitHub" + hosts = GITHUB_HOSTS + + def _headers(self) -> Dict[str, str]: + headers = {"Accept": "application/vnd.github+json"} + token = getattr(settings, "GITHUB_TOKEN", None) + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + # -- identifier ---------------------------------------------------------- # + def parse_identifier(self, value: str) -> str: + """Normalise to ``owner/repo`` or ``owner/repo@branch:sub/path``.""" + owner, repo, branch, path = self._parse(value) + ident = f"{owner}/{repo}" + if branch: + ident += f"@{branch}" + if path: + ident += f":{path}" + return ident + + def _parse(self, value: str) -> Tuple[str, str, Optional[str], str]: + raw = (value or "").strip() + if not raw: + raise InvalidIdentifierError("Enter a GitHub repository (owner/repo) or URL") + + branch: Optional[str] = None + path = "" + if "://" in raw or raw.startswith(GITHUB_HOSTS): + parsed = urlparse(raw if "://" in raw else f"https://{raw}") + if parsed.netloc.lower() not in GITHUB_HOSTS: + raise InvalidIdentifierError("That is not a github.com URL") + parts = [p for p in parsed.path.split("/") if p] + if len(parts) < 2: + raise InvalidIdentifierError( + "Expected a repository URL like https://github.com//" + ) + owner, repo = parts[0], parts[1].removesuffix(".git") + # /tree// or /blob// + if len(parts) >= 4 and parts[2] in ("tree", "blob"): + branch = parts[3] + path = "/".join(parts[4:]) + else: + spec = raw.strip("/") + if ":" in spec: + spec, path = spec.split(":", 1) + if "@" in spec: + spec, branch = spec.split("@", 1) + parts = spec.split("/") + if len(parts) != 2: + raise InvalidIdentifierError( + "GitHub repos look like 'owner/repo' (optionally owner/repo@branch:path)" + ) + owner, repo = parts + + if not (_NAME_RE.match(owner) and _NAME_RE.match(repo)): + raise InvalidIdentifierError( + "GitHub owner and repo names use letters, digits, '-', '_' and '.'" + ) + return owner, repo, (branch or None), path.strip("/") + + # -- fetch --------------------------------------------------------------- # + def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: + owner, repo, branch, sub_path = self._parse(identifier) + headers = self._headers() + + meta: Dict[str, Any] = self._get_json(f"{GITHUB_API}/repos/{owner}/{repo}", headers=headers) + if meta.get("disabled"): + raise PlatformImportError("This repository has been disabled on GitHub") + branch = branch or meta.get("default_branch") or "main" + full_name = meta.get("full_name") or f"{owner}/{repo}" + + # Repo names are slugs ("covid-19-data"); make a readable title from them. + title = repo.replace("-", " ").replace("_", " ").strip() + if sub_path: + title = f"{title} / {sub_path}" + + source_url = f"{GITHUB_WEB}/{full_name}" + if sub_path: + source_url += f"/tree/{branch}/{sub_path}" + + description = self._description(full_name, headers) or str(meta.get("description") or "") + license_info = meta.get("license") or {} + raw = dict(meta) + raw["_branch"] = branch + + return PlatformDatasetInfo( + platform=self.platform, + identifier=self.parse_identifier(identifier), + title=title[:300], + description=description[:MAX_DESCRIPTION], + source_url=source_url, + author=str((meta.get("owner") or {}).get("login") or owner)[:300], + license=str(license_info.get("spdx_id") or license_info.get("name") or ""), + tags=[t[:50] for t in (meta.get("topics") or []) if isinstance(t, str)][:30], + last_updated=parse_iso_datetime(meta.get("pushed_at") or meta.get("updated_at")), + raw=raw, + ) + + # -- pieces -------------------------------------------------------------- # + def _description(self, full_name: str, headers: Dict[str, str]) -> str: + try: + response = self._get( + f"{GITHUB_API}/repos/{full_name}/readme", + headers={**headers, "Accept": "application/vnd.github.raw"}, + ) + return response.text.strip() + except PlatformImportError: + return "" diff --git a/api/services/platform_importers/huggingface.py b/api/services/platform_importers/huggingface.py new file mode 100644 index 00000000..17b81c88 --- /dev/null +++ b/api/services/platform_importers/huggingface.py @@ -0,0 +1,165 @@ +"""Hugging Face Hub dataset importer (metadata only). + +Two requests per import: the repo metadata and the README (dataset card). +Public datasets need no token; ``HF_TOKEN`` (settings) is sent when present so +gated/private repos the token can see also work. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List +from urllib.parse import urlparse + +from django.conf import settings + +from api.services.platform_importers.base import ( + InvalidIdentifierError, + PlatformAuthError, + PlatformDatasetInfo, + PlatformImporter, + PlatformImportError, + parse_iso_datetime, +) +from api.utils.enums import ImportPlatform + +HF_HOSTS = ("huggingface.co", "www.huggingface.co", "hf.co") +HF_API = "https://huggingface.co/api/datasets" +HF_WEB = "https://huggingface.co/datasets" + +# "namespace/name" or a canonical "name"; HF ids allow letters, digits, - _ . +_ID_RE = re.compile(r"^(?:[A-Za-z0-9][A-Za-z0-9._-]*/)?[A-Za-z0-9][A-Za-z0-9._-]*$") + +MAX_DESCRIPTION = 1000 # Dataset.description column length + + +class HuggingFaceImporter(PlatformImporter): + platform = ImportPlatform.HUGGINGFACE + label = "Hugging Face" + hosts = HF_HOSTS + + def _headers(self) -> Dict[str, str]: + token = getattr(settings, "HF_TOKEN", None) + return {"Authorization": f"Bearer {token}"} if token else {} + + # -- identifier ---------------------------------------------------------- # + def parse_identifier(self, value: str) -> str: + raw = (value or "").strip() + if not raw: + raise InvalidIdentifierError("Enter a Hugging Face dataset id or URL") + + if "://" in raw or raw.startswith(tuple(HF_HOSTS)): + parsed = urlparse(raw if "://" in raw else f"https://{raw}") + if parsed.netloc.lower() not in HF_HOSTS: + raise InvalidIdentifierError("That is not a huggingface.co URL") + parts = [p for p in parsed.path.split("/") if p] + if not parts or parts[0] != "datasets": + raise InvalidIdentifierError( + "Expected a dataset URL like https://huggingface.co/datasets//" + ) + parts = parts[1:] + # Trim sub-paths such as /tree/main, /blob/main/..., /viewer. + if len(parts) >= 2 and parts[1] not in ( + "tree", + "blob", + "viewer", + "resolve", + "discussions", + ): + repo_id = f"{parts[0]}/{parts[1]}" + elif parts: + repo_id = parts[0] + else: + raise InvalidIdentifierError("Could not find a dataset id in that URL") + else: + repo_id = raw[len("datasets/") :] if raw.startswith("datasets/") else raw + + repo_id = repo_id.strip("/") + if not _ID_RE.match(repo_id): + raise InvalidIdentifierError( + "Hugging Face ids look like 'namespace/name' (letters, digits, '-', '_', '.')" + ) + return repo_id + + # -- fetch --------------------------------------------------------------- # + def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: + repo_id = self.parse_identifier(identifier) + headers = self._headers() + + try: + meta: Dict[str, Any] = self._get_json(f"{HF_API}/{repo_id}", headers=headers) + except PlatformAuthError as exc: + # Hugging Face answers 401 for repos that do not exist as well as + # for private/gated ones, so say both. + raise PlatformAuthError( + f"'{repo_id}' was not found on Hugging Face, or it is private/gated. " + "Check the spelling; only public datasets can be imported." + ) from exc + if meta.get("disabled"): + raise PlatformImportError("This dataset has been disabled on Hugging Face") + + card: Dict[str, Any] = meta.get("cardData") or {} + tags: List[str] = meta.get("tags") or [] + + title = card.get("pretty_name") or repo_id.split("/")[-1] + author = meta.get("author") or (repo_id.split("/")[0] if "/" in repo_id else "") + + return PlatformDatasetInfo( + platform=self.platform, + identifier=repo_id, + title=str(title)[:300], + description=self._description(repo_id, meta, headers), + source_url=f"{HF_WEB}/{repo_id}", + author=str(author)[:300], + license=self._license(card, tags), + tags=self._tags(tags), + last_updated=parse_iso_datetime(meta.get("lastModified")), + raw=meta, + ) + + # -- pieces -------------------------------------------------------------- # + @staticmethod + def _license(card: Dict[str, Any], tags: List[str]) -> str: + lic = card.get("license") + if isinstance(lic, list): + lic = lic[0] if lic else "" + if lic: + return str(lic) + for tag in tags: + if tag.startswith("license:"): + return tag[len("license:") :] + return "" + + @staticmethod + def _tags(tags: List[str]) -> List[str]: + """Keep human-meaningful tags: plain ones plus task/language values.""" + out: List[str] = [] + for tag in tags: + if ":" not in tag: + value = tag + else: + prefix, _, value = tag.partition(":") + if prefix not in ("task_categories", "language", "task_ids"): + continue + value = value.strip() + if value and value not in out: + out.append(value[:50]) + return out[:30] + + def _description(self, repo_id: str, meta: Dict[str, Any], headers: Dict[str, str]) -> str: + """Prefer the dataset card body (README) over the terse API field.""" + try: + response = self._get(f"{HF_WEB}/{repo_id}/resolve/main/README.md", headers=headers) + text = response.text + except PlatformImportError: + text = "" + if text: + # Strip YAML front matter. + if text.startswith("---"): + end = text.find("\n---", 3) + if end != -1: + text = text[end + 4 :] + text = text.strip() + if not text: + text = str(meta.get("description") or "") + return text[:MAX_DESCRIPTION] diff --git a/api/services/platform_importers/kaggle.py b/api/services/platform_importers/kaggle.py new file mode 100644 index 00000000..54bbef44 --- /dev/null +++ b/api/services/platform_importers/kaggle.py @@ -0,0 +1,121 @@ +"""Kaggle dataset importer (metadata only). + +One request per import: the dataset *view* endpoint, which answers anonymously +for public datasets and carries everything we store — title, description, +license, tags, owner and last updated. No key is needed. ``KAGGLE_USERNAME`` / +``KAGGLE_KEY`` (settings) are sent as basic auth when configured, which is only +useful for datasets the account can see but the public cannot. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse + +from django.conf import settings + +from api.services.platform_importers.base import ( + InvalidIdentifierError, + PlatformAuthError, + PlatformDatasetInfo, + PlatformImporter, + parse_iso_datetime, +) +from api.utils.enums import ImportPlatform + +KAGGLE_HOSTS = ("www.kaggle.com", "kaggle.com") +KAGGLE_API = "https://www.kaggle.com/api/v1" +KAGGLE_WEB = "https://www.kaggle.com/datasets" + +_SLUG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$") +MAX_DESCRIPTION = 1000 + + +class KaggleImporter(PlatformImporter): + platform = ImportPlatform.KAGGLE + label = "Kaggle" + hosts = KAGGLE_HOSTS + + def _auth(self) -> Optional[Tuple[str, str]]: + """Platform-level credentials if configured; None means anonymous.""" + username = getattr(settings, "KAGGLE_USERNAME", None) + key = getattr(settings, "KAGGLE_KEY", None) + return (username, key) if username and key else None + + # -- identifier ---------------------------------------------------------- # + def parse_identifier(self, value: str) -> str: + raw = (value or "").strip() + if not raw: + raise InvalidIdentifierError("Enter a Kaggle dataset ref (owner/dataset) or URL") + + if "://" in raw or raw.startswith(KAGGLE_HOSTS): + parsed = urlparse(raw if "://" in raw else f"https://{raw}") + if parsed.netloc.lower() not in KAGGLE_HOSTS: + raise InvalidIdentifierError("That is not a kaggle.com URL") + parts = [p for p in parsed.path.split("/") if p] + if parts and parts[0] == "datasets": + parts = parts[1:] + if len(parts) < 2: + raise InvalidIdentifierError( + "Expected a dataset URL like https://www.kaggle.com/datasets//" + ) + owner, slug = parts[0], parts[1] + else: + parts = raw.strip("/").split("/") + if len(parts) != 2: + raise InvalidIdentifierError("Kaggle refs look like 'owner/dataset-name'") + owner, slug = parts + + if not (_SLUG_RE.match(owner) and _SLUG_RE.match(slug)): + raise InvalidIdentifierError( + "Kaggle owner and dataset names use letters, digits, '-' and '_'" + ) + return f"{owner}/{slug}" + + # -- fetch --------------------------------------------------------------- # + def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: + ref = self.parse_identifier(identifier) + owner, slug = ref.split("/") + + meta: Dict[str, Any] = self._get_json( + f"{KAGGLE_API}/datasets/view/{owner}/{slug}", auth=self._auth() + ) + if meta.get("isPrivate"): + raise PlatformAuthError("This Kaggle dataset is private") + + title = meta.get("title") or slug + description = (meta.get("description") or meta.get("subtitle") or "").strip() + + return PlatformDatasetInfo( + platform=self.platform, + identifier=ref, + title=str(title)[:300], + description=description[:MAX_DESCRIPTION], + source_url=meta.get("url") or f"{KAGGLE_WEB}/{ref}", + author=str(meta.get("ownerName") or owner)[:300], + license=self._license(meta), + tags=self._tags(meta), + last_updated=parse_iso_datetime(meta.get("lastUpdated")), + raw=meta, + ) + + # -- pieces -------------------------------------------------------------- # + @staticmethod + def _license(meta: Dict[str, Any]) -> str: + licenses = meta.get("licenses") or [] + if licenses and isinstance(licenses[0], dict): + return str(licenses[0].get("name") or "") + return str(meta.get("licenseName") or "") + + @staticmethod + def _tags(meta: Dict[str, Any]) -> List[str]: + out: List[str] = [] + for entry in meta.get("keywords") or []: + if isinstance(entry, str) and entry.strip(): + out.append(entry.strip()[:50]) + for entry in meta.get("tags") or []: + name = entry.get("name") if isinstance(entry, dict) else entry + if isinstance(name, str) and name.strip() and name.strip() not in out: + out.append(name.strip()[:50]) + return out[:30] diff --git a/api/types/type_dataset.py b/api/types/type_dataset.py index b8e71ec3..dcc0ed3e 100644 --- a/api/types/type_dataset.py +++ b/api/types/type_dataset.py @@ -11,6 +11,7 @@ from api.models import Dataset, DatasetMetadata, PromptDataset, Resource, Tag from api.types.base_type import BaseType from api.types.type_dataset_metadata import TypeDatasetMetadata +from api.types.type_dataset_source import TypeDatasetSource from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization from api.types.type_resource import TypeResource @@ -65,6 +66,12 @@ class TypeDataset(BaseType): download_count: int user: Optional["TypeUser"] + @strawberry.field + def source(self) -> Optional["TypeDatasetSource"]: + """Provenance for datasets imported from a third-party platform (else null).""" + source = getattr(self, "source", None) + return TypeDatasetSource.from_django(source) if source is not None else None + @strawberry.field def sectors(self, info: Info) -> List["TypeSector"]: """Get sectors for this dataset. diff --git a/api/types/type_dataset_source.py b/api/types/type_dataset_source.py new file mode 100644 index 00000000..1655c2fb --- /dev/null +++ b/api/types/type_dataset_source.py @@ -0,0 +1,62 @@ +"""GraphQL types for third-party platform imports (link-only).""" + +from datetime import datetime +from typing import List, Optional + +import strawberry +import strawberry_django +from strawberry import auto +from strawberry.enum import EnumType + +from api.models import DatasetSource +from api.services.platform_importers import PlatformDatasetInfo +from api.types.base_type import BaseType +from api.utils.enums import ImportPlatform + +import_platform_enum: EnumType = strawberry.enum(ImportPlatform) # type: ignore + + +@strawberry_django.type(DatasetSource) +class TypeDatasetSource(BaseType): + """Where an imported dataset came from.""" + + id: auto + platform: import_platform_enum # type: ignore + source_identifier: auto + source_url: auto + source_author: auto + source_license: auto + source_last_updated: auto + imported_at: auto + last_synced_at: auto + + +@strawberry.type +class TypePlatformDatasetPreview: + """What an import *would* create — returned by the preview query, no side effects.""" + + platform: import_platform_enum # type: ignore + identifier: str + title: str + description: str + source_url: str + author: str + license: str + mapped_license: str + tags: List[str] + last_updated: Optional[datetime] + + @classmethod + def from_info(cls, info: PlatformDatasetInfo) -> "TypePlatformDatasetPreview": + return cls( + platform=ImportPlatform(info.platform), + identifier=info.identifier, + title=info.title, + description=info.description, + source_url=info.source_url, + author=info.author, + license=info.license, + mapped_license=info.mapped_license, + tags=list(info.tags), + last_updated=info.last_updated, + ) diff --git a/api/types/type_resource.py b/api/types/type_resource.py index cd6d09c3..e77d351e 100644 --- a/api/types/type_resource.py +++ b/api/types/type_resource.py @@ -68,6 +68,7 @@ class TypeResource(BaseType): preview_enabled: auto preview_details: Optional[TypePreviewDetails] download_count: auto + url: auto # @strawberry.field # def model_resources(self) -> List[TypeAccessModelResourceFields]: diff --git a/api/utils/enums.py b/api/utils/enums.py index 646749ff..49e1a610 100644 --- a/api/utils/enums.py +++ b/api/utils/enums.py @@ -324,3 +324,11 @@ class EndpointAuthType(models.TextChoices): OAUTH2 = "OAUTH2" CUSTOM = "CUSTOM" NONE = "NONE" + + +class ImportPlatform(models.TextChoices): + """Third-party platforms a dataset can be imported (link-only) from.""" + + KAGGLE = "KAGGLE" + HUGGINGFACE = "HUGGINGFACE" + GITHUB = "GITHUB" diff --git a/api/views/download_view.py b/api/views/download_view.py index cce801a0..5d106483 100644 --- a/api/views/download_view.py +++ b/api/views/download_view.py @@ -6,7 +6,7 @@ from asgiref.sync import sync_to_async from django.core.exceptions import ObjectDoesNotExist from django.core.files.uploadedfile import UploadedFile -from django.http import HttpRequest, HttpResponse, JsonResponse +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse from pyecharts.charts.chart import Chart from pyecharts.render import make_snapshot from selenium import webdriver @@ -16,6 +16,7 @@ from api.models import Resource, ResourceChartDetails, ResourceChartImage from api.types.type_resource_chart import chart_base +from api.utils.enums import DataType @sync_to_async @@ -47,13 +48,21 @@ def get_resource_response( resource: Resource, request: Optional[HttpRequest] = None ) -> HttpResponse: """Get file response for a resource.""" - file_details = resource.resourcefiledetails + # Link-only (platform-imported) resources: we hold no bytes, send the + # user to the file on the source platform. Still counts as a download. + if resource.type == DataType.EXTERNAL: + if not resource.url: + return JsonResponse({"error": "External resource has no URL"}, status=404) + resource.download_count += 1 + resource.save(update_fields=["download_count"]) + _track_download(resource, request) + return HttpResponseRedirect(resource.url) + + file_details = getattr(resource, "resourcefiledetails", None) if not file_details or not file_details.file: return JsonResponse({"error": "File not found"}, status=404) - response = HttpResponse( - file_details.file.read(), content_type="application/octet-stream" - ) + response = HttpResponse(file_details.file.read(), content_type="application/octet-stream") # Handle filename and basename explicitly default_name = f"resource_{resource.name}.csv" @@ -69,7 +78,14 @@ def get_resource_response( resource.download_count += 1 resource.save() - # Track the download activity if the user is authenticated + _track_download(resource, request) + + response["Content-Disposition"] = f'attachment; filename="{basename}"' + return response + + +def _track_download(resource: Resource, request: Optional[HttpRequest]) -> None: + """Record the download in the activity stream for authenticated users.""" if request and hasattr(request, "user") and request.user.is_authenticated: # Import here to avoid circular imports import asyncio @@ -80,9 +96,6 @@ def get_resource_response( sync_to_async(track_resource_downloaded)(request.user, resource, request) ) - response["Content-Disposition"] = f'attachment; filename="{basename}"' - return response - @sync_to_async def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponse: @@ -90,9 +103,7 @@ def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponse: if not chart_image.image: return JsonResponse({"error": "File not found"}, status=404) - response = HttpResponse( - chart_image.image.read(), content_type="application/octet-stream" - ) + response = HttpResponse(chart_image.image.read(), content_type="application/octet-stream") # Handle filename and basename explicitly default_name = f"chart_{chart_image.id}.png" @@ -180,9 +191,7 @@ def get_file_chart_image_response(chart_image: ResourceChartImage) -> HttpRespon file_obj.seek(0) # Reset file pointer response = HttpResponse(file_obj, content_type=mime_type) file_name = str(file_obj.name) - response["Content-Disposition"] = ( - f'attachment; filename="{os.path.basename(file_name)}"' - ) + response["Content-Disposition"] = f'attachment; filename="{os.path.basename(file_name)}"' else: response = HttpResponse("File doesn't exist", content_type="text/plain") return response @@ -192,9 +201,7 @@ def get_custom_webdriver() -> WebDriver: """Configure and return a custom Selenium WebDriver.""" chrome_options = Options() chrome_options.add_argument("--no-sandbox") # Bypass OS security model - chrome_options.add_argument( - "--disable-dev-shm-usage" - ) # Overcome limited resource problems + chrome_options.add_argument("--disable-dev-shm-usage") # Overcome limited resource problems chrome_options.add_argument("--headless") # Run headless browser chrome_options.add_argument("--disable-gpu") # Disable GPU for headless browser diff --git a/api/views/search_dataset.py b/api/views/search_dataset.py index 680d5093..b58b8115 100644 --- a/api/views/search_dataset.py +++ b/api/views/search_dataset.py @@ -81,6 +81,7 @@ class DatasetDocumentSerializer(serializers.ModelSerializer): tags = serializers.ListField() sectors = serializers.ListField() formats = serializers.ListField() + source_platform = serializers.CharField(required=False, allow_null=True) catalogs = serializers.ListField() geographies = serializers.ListField() has_charts = serializers.BooleanField() @@ -118,6 +119,7 @@ class Meta: "tags", "sectors", "formats", + "source_platform", "catalogs", "geographies", "has_charts", @@ -175,6 +177,7 @@ def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: "catalogs.raw": "terms", "geographies.raw": "terms", "dataset_type": "terms", + "source_platform": "terms", } for metadata in enabled_metadata: # type: Metadata if metadata.filterable: @@ -273,6 +276,13 @@ def add_filters(self, filters: Dict[str, str], search: Search) -> Search: elif filter == "dataset_type": # Filter by dataset type (DATA or PROMPT) search = search.filter("term", dataset_type=filters[filter]) + elif filter == "source_platform": + # Filter by import platform (HUGGINGFACE, GITHUB, KAGGLE); "NATIVE" + # selects datasets that were not imported at all. + if filters[filter] == "NATIVE": + search = search.exclude("exists", field="source_platform") + else: + search = search.filter("terms", source_platform=filters[filter].split(",")) elif filter == "task_type": # Filter by prompt task type (nested in prompt_metadata) search = search.filter( diff --git a/api/views/search_unified.py b/api/views/search_unified.py index e718d580..1323f5ec 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -64,6 +64,7 @@ class UserSerializer(serializers.Serializer): # Type-specific fields # Dataset specific formats = serializers.ListField(required=False) + source_platform = serializers.CharField(required=False, allow_null=True) has_charts = serializers.BooleanField(required=False) download_count = serializers.IntegerField(required=False) is_individual_dataset = serializers.BooleanField(required=False) diff --git a/search/documents/dataset_document.py b/search/documents/dataset_document.py index 83f8d6e7..559d5cf5 100644 --- a/search/documents/dataset_document.py +++ b/search/documents/dataset_document.py @@ -6,6 +6,7 @@ Catalog, Dataset, DatasetMetadata, + DatasetSource, Geography, Metadata, Organization, @@ -94,6 +95,10 @@ class DatasetDocument(Document): } ) + # Platform this dataset was imported from (KAGGLE, HUGGINGFACE) or null + # for datasets created natively. Lets listings badge/filter imports. + source_platform = fields.KeywordField(attr="source_platform_indexing") + formats = fields.TextField( attr="formats_indexing", analyzer=ngram_analyser, @@ -238,6 +243,8 @@ def get_instances_from_related( """Get Dataset instances from related models.""" if isinstance(related_instance, Resource): return related_instance.dataset + elif isinstance(related_instance, DatasetSource): + return related_instance.dataset elif isinstance(related_instance, Metadata): ds_metadata_objects = related_instance.datasetmetadata_set.all() return [obj.dataset for obj in ds_metadata_objects] # type: ignore @@ -271,6 +278,7 @@ class Django: related_models = [ Resource, + DatasetSource, Metadata, DatasetMetadata, PromptDataset, From 92e8487523296b897fbe7cd89898cbe3ed4f5f21 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 22 Sep 2026 01:56:05 +0530 Subject: [PATCH 2/4] fix(api): do not download or store the Hugging Face file list on import The Hub's dataset endpoint returns `siblings`, one entry per file, by default. For large repos that key dominates the response: 9.6 MB for an 85k-file repo against ~4 KB for everything else. The importer stored the whole response in DatasetSource.raw_metadata and fetched it on every preview and import, although files are never listed. Request fields by name with `expand[]` (every default field except `siblings`, plus `citation`), and drop `siblings` defensively before the payload is kept. --- .../platform_importers/huggingface.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/api/services/platform_importers/huggingface.py b/api/services/platform_importers/huggingface.py index 17b81c88..2ab4a934 100644 --- a/api/services/platform_importers/huggingface.py +++ b/api/services/platform_importers/huggingface.py @@ -1,6 +1,7 @@ """Hugging Face Hub dataset importer (metadata only). -Two requests per import: the repo metadata and the README (dataset card). +Two small requests per import: the repo metadata (without its file list) and +the README (dataset card). Public datasets need no token; ``HF_TOKEN`` (settings) is sent when present so gated/private repos the token can see also work. """ @@ -32,6 +33,28 @@ MAX_DESCRIPTION = 1000 # Dataset.description column length +# Everything the Hub returns by default EXCEPT ``siblings`` (the per-file list). +# We never list files, and for large repos that one key is most of the payload: +# ~9.6 MB for an 85k-file repo versus ~4 KB without it. Asking for fields by +# name means the list is never downloaded, and never stored in raw_metadata. +_EXPAND_FIELDS = ( + "author", + "cardData", + "citation", + "createdAt", + "description", + "disabled", + "downloads", + "gated", + "lastModified", + "likes", + "paperswithcode_id", + "private", + "sha", + "tags", + "usedStorage", +) + class HuggingFaceImporter(PlatformImporter): platform = ImportPlatform.HUGGINGFACE @@ -87,7 +110,11 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: headers = self._headers() try: - meta: Dict[str, Any] = self._get_json(f"{HF_API}/{repo_id}", headers=headers) + meta: Dict[str, Any] = self._get_json( + f"{HF_API}/{repo_id}", + headers=headers, + params=[("expand[]", name) for name in _EXPAND_FIELDS], + ) except PlatformAuthError as exc: # Hugging Face answers 401 for repos that do not exist as well as # for private/gated ones, so say both. @@ -95,6 +122,7 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: f"'{repo_id}' was not found on Hugging Face, or it is private/gated. " "Check the spelling; only public datasets can be imported." ) from exc + meta.pop("siblings", None) # never keep the file list, whatever the API sends if meta.get("disabled"): raise PlatformImportError("This dataset has been disabled on Hugging Face") From 158f2656cce312e9003611794694f23e513120a5 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 24 Sep 2026 02:11:22 +0530 Subject: [PATCH 3/4] feat(api): store imported metadata in typed columns, drop the raw payload DatasetSource no longer keeps the platform's raw JSON. Every field we fetch now lands in a typed column, chosen by one rule: a metadata standard reads it on export (DCAT / Croissant / Dublin Core) or the platform itself reads it (attribution, duplicate check, licence review). New columns: revision (commit hash or version), source_created_at, source_readme (full card; Dataset.description keeps a 1,000-char cut), citation, languages, source_homepage, is_archived. Column definitions a platform declares (Hugging Face dataset_info) become ResourceSchema rows on the link resource, so the columns list works without fetching data. Importers fill what each platform provides: Hugging Face all of the above; GitHub adds one small call for the branch head SHA and reads homepage/archived; Kaggle uses the version number and the earliest version date. Fields nothing reads (downloads, likes, stars, size categories, task taxonomy) are no longer fetched. --- api/migrations/0048_platform_import.py | 10 +++- api/models/DatasetSource.py | 33 ++++++++++-- api/services/platform_import_service.py | 21 +++++++- api/services/platform_importers/__init__.py | 2 + api/services/platform_importers/base.py | 48 ++++++++++++++++- api/services/platform_importers/github.py | 30 ++++++++--- .../platform_importers/huggingface.py | 53 +++++++++++++++---- api/services/platform_importers/kaggle.py | 21 ++++++-- api/types/type_dataset_source.py | 18 ++++++- 9 files changed, 203 insertions(+), 33 deletions(-) diff --git a/api/migrations/0048_platform_import.py b/api/migrations/0048_platform_import.py index ac2e5107..9efe2d02 100644 --- a/api/migrations/0048_platform_import.py +++ b/api/migrations/0048_platform_import.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.4 on 2026-09-20 10:13 +# Generated by Django 5.0.4 on 2026-09-23 20:36 import uuid @@ -37,10 +37,16 @@ class Migration(migrations.Migration): ), ("source_identifier", models.CharField(max_length=300)), ("source_url", models.URLField(max_length=500)), + ("source_homepage", models.URLField(blank=True, max_length=500)), + ("revision", models.CharField(blank=True, max_length=64)), ("source_author", models.CharField(blank=True, max_length=300)), ("source_license", models.CharField(blank=True, max_length=300)), + ("source_readme", models.TextField(blank=True)), + ("citation", models.TextField(blank=True)), + ("languages", models.JSONField(blank=True, default=list)), + ("source_created_at", models.DateTimeField(blank=True, null=True)), ("source_last_updated", models.DateTimeField(blank=True, null=True)), - ("raw_metadata", models.JSONField(blank=True, default=dict)), + ("is_archived", models.BooleanField(default=False)), ("imported_at", models.DateTimeField(auto_now_add=True)), ("last_synced_at", models.DateTimeField(auto_now=True)), ( diff --git a/api/models/DatasetSource.py b/api/models/DatasetSource.py index e9aa7817..ccf0b2ad 100644 --- a/api/models/DatasetSource.py +++ b/api/models/DatasetSource.py @@ -9,25 +9,48 @@ class DatasetSource(models.Model): """Provenance record for a dataset imported from a third-party platform. Imports are link-only: DataSpace never copies the platform's files. This - row remembers where the dataset came from so the UI can attribute it, link - back to it, and (later) re-sync its metadata. The raw platform response is - kept in ``raw_metadata`` for debugging and future field mapping. + row holds what the platform told us about the dataset, in typed columns. + Every column here has a reader: either a metadata standard on export + (DCAT / Croissant / Dublin Core) or the platform itself (attribution, + duplicate detection, licence review). No raw payload is kept. """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) dataset = models.OneToOneField("api.Dataset", on_delete=models.CASCADE, related_name="source") platform = models.CharField(max_length=50, choices=ImportPlatform.choices) + + # --- identity on the platform ------------------------------------------ # Platform-native identifier, e.g. "owner/dataset-slug" (Kaggle) or # "namespace/name" (Hugging Face). Normalised by the importer. source_identifier = models.CharField(max_length=300) - # Human-facing page on the platform. + # Human-facing page on the platform. Export: schema:sameAs / prov:wasDerivedFrom. source_url = models.URLField(max_length=500) + # Home page declared by the source, if any (GitHub `homepage`). Export: dcat:landingPage. + source_homepage = models.URLField(max_length=500, blank=True) + # Commit hash (Hugging Face / GitHub) or version number (Kaggle) at import time. + # Export: Croissant `version`. Later: what a sync compares against. + revision = models.CharField(max_length=64, blank=True) + + # --- descriptive metadata the standards read ---------------------------- + # Who made the data on the platform. Export: dcterms:creator / Croissant creator. source_author = models.CharField(max_length=300, blank=True) # License string exactly as the platform reported it (may not map onto # DatasetLicense; the mapped value lives on Dataset.license). source_license = models.CharField(max_length=300, blank=True) + # Full dataset card / README. Dataset.description keeps a 1,000-char cut. + source_readme = models.TextField(blank=True) + # BibTeX or free-text citation, when the platform provides one. Export: Croissant citeAs. + citation = models.TextField(blank=True) + # Language codes of the data, e.g. ["en", "hi"]. Export: dcterms:language / inLanguage. + languages = models.JSONField(default=list, blank=True) + # When the dataset was first published on the platform. Export: dcterms:issued. + source_created_at = models.DateTimeField(null=True, blank=True) + # When the platform last changed it. Export: dcterms:modified. source_last_updated = models.DateTimeField(null=True, blank=True) - raw_metadata = models.JSONField(default=dict, blank=True) + # Source is frozen / read-only upstream (GitHub `archived`). Shown as a hint. + is_archived = models.BooleanField(default=False) + + # --- our side ------------------------------------------------------------- imported_by = models.ForeignKey( "authorization.User", on_delete=models.SET_NULL, diff --git a/api/services/platform_import_service.py b/api/services/platform_import_service.py index b30fe17b..e60e2304 100644 --- a/api/services/platform_import_service.py +++ b/api/services/platform_import_service.py @@ -24,6 +24,7 @@ Metadata, Organization, Resource, + ResourceSchema, Sector, Tag, ) @@ -202,7 +203,7 @@ def import_platform_dataset( # the dataset page, so "download" redirects there and people browse/fetch # files with the platform's own tooling. platform_label = PLATFORM_LABELS.get(str(info.platform), str(info.platform).title()) - Resource.objects.create( + link_resource = Resource.objects.create( dataset=dataset, type=DataType.EXTERNAL, name=f"Dataset on {platform_label}"[:200], @@ -215,13 +216,29 @@ def import_platform_dataset( platform=info.platform, source_identifier=info.identifier, source_url=info.source_url[:500], + source_homepage=(info.homepage or "")[:500], + revision=(info.revision or "")[:64], source_author=info.author[:300], source_license=info.license[:300], + source_readme=info.readme or "", + citation=info.citation or "", + languages=list(info.languages or []), + source_created_at=info.created_at, source_last_updated=info.last_updated, - raw_metadata=info.raw, + is_archived=bool(info.is_archived), imported_by=user, ) + # Column definitions the platform declared (Hugging Face dataset_info). + # They live on the link resource so "View All Columns" and the Croissant + # recordSet work without any data being fetched. + ResourceSchema.objects.bulk_create( + [ + ResourceSchema(resource=link_resource, field_name=col.name, format=col.field_type) + for col in info.columns + ] + ) + try: owner_role = Role.objects.get(name="owner") except Role.DoesNotExist as exc: diff --git a/api/services/platform_importers/__init__.py b/api/services/platform_importers/__init__.py index 246c6b08..be240499 100644 --- a/api/services/platform_importers/__init__.py +++ b/api/services/platform_importers/__init__.py @@ -10,6 +10,7 @@ from api.services.platform_importers.base import ( InvalidIdentifierError, PlatformAuthError, + PlatformColumn, PlatformDatasetInfo, PlatformDatasetNotFoundError, PlatformImporter, @@ -53,6 +54,7 @@ def detect_platform(value: str) -> Optional[str]: "detect_platform", "PlatformImporter", "PlatformDatasetInfo", + "PlatformColumn", "PlatformImportError", "InvalidIdentifierError", "PlatformDatasetNotFoundError", diff --git a/api/services/platform_importers/base.py b/api/services/platform_importers/base.py index bc6a671a..3efcd360 100644 --- a/api/services/platform_importers/base.py +++ b/api/services/platform_importers/base.py @@ -53,18 +53,36 @@ class PlatformUnavailableError(PlatformImportError): # --------------------------------------------------------------------------- # # Normalised result # --------------------------------------------------------------------------- # +@dataclass +class PlatformColumn: + """One column of the dataset as the platform describes it.""" + + name: str + field_type: str # a FieldTypes value: STRING / NUMBER / INTEGER / DATE / BOOLEAN + + @dataclass class PlatformDatasetInfo: + """Everything an importer returns. Each field lands in a typed column on + DatasetSource (or on Dataset / ResourceSchema); nothing raw is kept.""" + platform: str identifier: str title: str - description: str + description: str # short form, fits Dataset.description (1,000 chars) source_url: str author: str = "" license: str = "" tags: List[str] = field(default_factory=list) last_updated: Optional[datetime] = None - raw: Dict[str, Any] = field(default_factory=dict) + created_at: Optional[datetime] = None + revision: str = "" + readme: str = "" # full card / README, unbounded + citation: str = "" + languages: List[str] = field(default_factory=list) + homepage: str = "" + is_archived: bool = False + columns: List[PlatformColumn] = field(default_factory=list) @property def mapped_license(self) -> str: @@ -103,6 +121,29 @@ def map_license(raw: str) -> str: return LICENSE_ALIASES.get(key, DatasetLicense.CC_BY_4_0_ATTRIBUTION) +def field_type_for(dtype: Any) -> str: + """Map a platform column type (Hugging Face / Arrow style names) onto FieldTypes.""" + name = str(dtype if isinstance(dtype, str) else "").lower() + if name in ("bool", "boolean"): + return "BOOLEAN" + if name.startswith(("int", "uint")): + return "INTEGER" + if name.startswith(("float", "double", "decimal")): + return "NUMBER" + if name.startswith(("date", "timestamp", "time")): + return "DATE" + return "STRING" # strings, class labels, nested/binary types + + +def shorten(text: str, limit: int = 1000) -> str: + """Cut on a word boundary with an ellipsis; used for Dataset.description.""" + text = (text or "").strip() + if len(text) <= limit: + return text + cut = text[: limit - 1].rsplit(" ", 1)[0] + return cut + "…" + + def parse_iso_datetime(value: Optional[str]) -> Optional[datetime]: """Parse a platform timestamp. Django's parser copes with a trailing ``Z`` and any number of fractional digits (Kaggle sends e.g. ``...:04.7Z``), @@ -186,11 +227,14 @@ def _get_json(self, url: str, **kwargs: Any) -> Any: "ImportPlatform", "PlatformImporter", "PlatformDatasetInfo", + "PlatformColumn", "PlatformImportError", "InvalidIdentifierError", "PlatformDatasetNotFoundError", "PlatformAuthError", "PlatformUnavailableError", "map_license", + "field_type_for", + "shorten", "parse_iso_datetime", ] diff --git a/api/services/platform_importers/github.py b/api/services/platform_importers/github.py index c5202e31..8a525228 100644 --- a/api/services/platform_importers/github.py +++ b/api/services/platform_importers/github.py @@ -1,7 +1,8 @@ """GitHub repository importer (metadata only). -A repo (optionally a branch and sub-folder) is treated as a dataset. Two -requests per import: the repo metadata and the README. Public repos need no +A repo (optionally a branch and sub-folder) is treated as a dataset. Three +small requests per import: the repo metadata, the branch head (for the +revision) and the README. Public repos need no token; ``GITHUB_TOKEN`` (settings) is sent when present, mainly to lift the anonymous 60 requests/hour rate limit. """ @@ -20,6 +21,7 @@ PlatformImporter, PlatformImportError, parse_iso_datetime, + shorten, ) from api.utils.enums import ImportPlatform @@ -114,26 +116,28 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: if sub_path: source_url += f"/tree/{branch}/{sub_path}" - description = self._description(full_name, headers) or str(meta.get("description") or "") + readme = self._readme(full_name, headers) or str(meta.get("description") or "") license_info = meta.get("license") or {} - raw = dict(meta) - raw["_branch"] = branch return PlatformDatasetInfo( platform=self.platform, identifier=self.parse_identifier(identifier), title=title[:300], - description=description[:MAX_DESCRIPTION], + description=shorten(readme, MAX_DESCRIPTION), source_url=source_url, author=str((meta.get("owner") or {}).get("login") or owner)[:300], license=str(license_info.get("spdx_id") or license_info.get("name") or ""), tags=[t[:50] for t in (meta.get("topics") or []) if isinstance(t, str)][:30], last_updated=parse_iso_datetime(meta.get("pushed_at") or meta.get("updated_at")), - raw=raw, + created_at=parse_iso_datetime(meta.get("created_at")), + revision=self._head_sha(full_name, branch, headers), + readme=readme, + homepage=str(meta.get("homepage") or "")[:500], + is_archived=bool(meta.get("archived")), ) # -- pieces -------------------------------------------------------------- # - def _description(self, full_name: str, headers: Dict[str, str]) -> str: + def _readme(self, full_name: str, headers: Dict[str, str]) -> str: try: response = self._get( f"{GITHUB_API}/repos/{full_name}/readme", @@ -142,3 +146,13 @@ def _description(self, full_name: str, headers: Dict[str, str]) -> str: return response.text.strip() except PlatformImportError: return "" + + def _head_sha(self, full_name: str, branch: str, headers: Dict[str, str]) -> str: + """Commit SHA at the branch head; empty if the lookup fails.""" + try: + ref: Dict[str, Any] = self._get_json( + f"{GITHUB_API}/repos/{full_name}/git/ref/heads/{branch}", headers=headers + ) + return str((ref.get("object") or {}).get("sha") or "")[:64] + except PlatformImportError: + return "" diff --git a/api/services/platform_importers/huggingface.py b/api/services/platform_importers/huggingface.py index 2ab4a934..bf4ea994 100644 --- a/api/services/platform_importers/huggingface.py +++ b/api/services/platform_importers/huggingface.py @@ -17,10 +17,13 @@ from api.services.platform_importers.base import ( InvalidIdentifierError, PlatformAuthError, + PlatformColumn, PlatformDatasetInfo, PlatformImporter, PlatformImportError, + field_type_for, parse_iso_datetime, + shorten, ) from api.utils.enums import ImportPlatform @@ -36,7 +39,7 @@ # Everything the Hub returns by default EXCEPT ``siblings`` (the per-file list). # We never list files, and for large repos that one key is most of the payload: # ~9.6 MB for an 85k-file repo versus ~4 KB without it. Asking for fields by -# name means the list is never downloaded, and never stored in raw_metadata. +# name means the list is never downloaded, and never stored anywhere. _EXPAND_FIELDS = ( "author", "cardData", @@ -44,15 +47,11 @@ "createdAt", "description", "disabled", - "downloads", "gated", "lastModified", - "likes", - "paperswithcode_id", "private", "sha", "tags", - "usedStorage", ) @@ -131,18 +130,24 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: title = card.get("pretty_name") or repo_id.split("/")[-1] author = meta.get("author") or (repo_id.split("/")[0] if "/" in repo_id else "") + readme = self._readme(repo_id, meta, headers) return PlatformDatasetInfo( platform=self.platform, identifier=repo_id, title=str(title)[:300], - description=self._description(repo_id, meta, headers), + description=shorten(readme, MAX_DESCRIPTION), source_url=f"{HF_WEB}/{repo_id}", author=str(author)[:300], license=self._license(card, tags), tags=self._tags(tags), last_updated=parse_iso_datetime(meta.get("lastModified")), - raw=meta, + created_at=parse_iso_datetime(meta.get("createdAt")), + revision=str(meta.get("sha") or "")[:64], + readme=readme, + citation=str(meta.get("citation") or ""), + languages=self._languages(card, tags), + columns=self._columns(card), ) # -- pieces -------------------------------------------------------------- # @@ -174,8 +179,8 @@ def _tags(tags: List[str]) -> List[str]: out.append(value[:50]) return out[:30] - def _description(self, repo_id: str, meta: Dict[str, Any], headers: Dict[str, str]) -> str: - """Prefer the dataset card body (README) over the terse API field.""" + def _readme(self, repo_id: str, meta: Dict[str, Any], headers: Dict[str, str]) -> str: + """Full dataset card body (README) without its YAML front matter; else the API field.""" try: response = self._get(f"{HF_WEB}/{repo_id}/resolve/main/README.md", headers=headers) text = response.text @@ -190,4 +195,32 @@ def _description(self, repo_id: str, meta: Dict[str, Any], headers: Dict[str, st text = text.strip() if not text: text = str(meta.get("description") or "") - return text[:MAX_DESCRIPTION] + return text + + @staticmethod + def _languages(card: Dict[str, Any], tags: List[str]) -> List[str]: + langs = card.get("language") + if isinstance(langs, str): + langs = [langs] + out = [str(v).strip() for v in (langs or []) if str(v).strip()] + if not out: + out = [t[len("language:") :] for t in tags if t.startswith("language:")] + return out[:20] + + @staticmethod + def _columns(card: Dict[str, Any]) -> List[PlatformColumn]: + """Column names/types from the card's dataset_info (first config if several).""" + info = card.get("dataset_info") + if isinstance(info, list): + info = info[0] if info else None + if not isinstance(info, dict): + return [] + cols: List[PlatformColumn] = [] + for feat in info.get("features") or []: + if isinstance(feat, dict) and feat.get("name"): + cols.append( + PlatformColumn( + name=str(feat["name"])[:255], field_type=field_type_for(feat.get("dtype")) + ) + ) + return cols[:200] diff --git a/api/services/platform_importers/kaggle.py b/api/services/platform_importers/kaggle.py index 54bbef44..bb831254 100644 --- a/api/services/platform_importers/kaggle.py +++ b/api/services/platform_importers/kaggle.py @@ -21,6 +21,7 @@ PlatformDatasetInfo, PlatformImporter, parse_iso_datetime, + shorten, ) from api.utils.enums import ImportPlatform @@ -85,19 +86,22 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: raise PlatformAuthError("This Kaggle dataset is private") title = meta.get("title") or slug - description = (meta.get("description") or meta.get("subtitle") or "").strip() + readme = (meta.get("description") or meta.get("subtitle") or "").strip() + version = meta.get("currentVersionNumber") return PlatformDatasetInfo( platform=self.platform, identifier=ref, title=str(title)[:300], - description=description[:MAX_DESCRIPTION], + description=shorten(readme, MAX_DESCRIPTION), source_url=meta.get("url") or f"{KAGGLE_WEB}/{ref}", author=str(meta.get("ownerName") or owner)[:300], license=self._license(meta), tags=self._tags(meta), last_updated=parse_iso_datetime(meta.get("lastUpdated")), - raw=meta, + created_at=self._first_version_date(meta), + revision=str(version) if version is not None else "", + readme=readme, ) # -- pieces -------------------------------------------------------------- # @@ -119,3 +123,14 @@ def _tags(meta: Dict[str, Any]) -> List[str]: if isinstance(name, str) and name.strip() and name.strip() not in out: out.append(name.strip()[:50]) return out[:30] + + @staticmethod + def _first_version_date(meta: Dict[str, Any]): + """Kaggle has no created date; the earliest version's creation date is the same thing.""" + dates = [ + parse_iso_datetime(v.get("creationDate")) + for v in (meta.get("versions") or []) + if isinstance(v, dict) + ] + dates = [d for d in dates if d is not None] + return min(dates) if dates else None diff --git a/api/types/type_dataset_source.py b/api/types/type_dataset_source.py index 1655c2fb..00c62261 100644 --- a/api/types/type_dataset_source.py +++ b/api/types/type_dataset_source.py @@ -18,18 +18,28 @@ @strawberry_django.type(DatasetSource) class TypeDatasetSource(BaseType): - """Where an imported dataset came from.""" + """Where an imported dataset came from, and what the platform said about it.""" id: auto platform: import_platform_enum # type: ignore source_identifier: auto source_url: auto + source_homepage: auto + revision: auto source_author: auto source_license: auto + source_readme: auto + citation: auto + source_created_at: auto source_last_updated: auto + is_archived: auto imported_at: auto last_synced_at: auto + @strawberry.field + def languages(self) -> List[str]: + return list(getattr(self, "languages", None) or []) + @strawberry.type class TypePlatformDatasetPreview: @@ -45,6 +55,9 @@ class TypePlatformDatasetPreview: mapped_license: str tags: List[str] last_updated: Optional[datetime] + languages: List[str] + revision: str + column_count: int @classmethod def from_info(cls, info: PlatformDatasetInfo) -> "TypePlatformDatasetPreview": @@ -59,4 +72,7 @@ def from_info(cls, info: PlatformDatasetInfo) -> "TypePlatformDatasetPreview": mapped_license=info.mapped_license, tags=list(info.tags), last_updated=info.last_updated, + languages=list(info.languages), + revision=info.revision, + column_count=len(info.columns), ) From f7fe195aa39340192c4d22c78a6bf76a7564aed4 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 24 Sep 2026 19:08:08 +0530 Subject: [PATCH 4/4] fix(api): harden platform import after a live probe of 36 datasets Found by importing many real datasets and fuzzing identifiers: - Hugging Face: keep the platform's canonical id ("imdb" is really "stanfordnlp/imdb"), so the same dataset cannot be imported twice under a legacy name. Duplicate check re-runs under the canonical id. - GitHub: SPDX "NOASSERTION"/"other" means no detectable licence; treat as empty instead of a licence called NOASSERTION. Branch names and folder paths are validated (no "..", no whitespace, plain segments only). - Kaggle: a 403 covers "does not exist" as well as private, so say both. The created date is taken from the earliest version only when the view lists every version (it lists one of 2,324 for kaggle/meta-kaggle). - All: identifiers over 200 characters are rejected; README kept to 200 KB; citation to 20 KB. - Service: platform calls now happen before the transaction is opened, so a slow platform never holds a database connection. --- api/services/platform_import_service.py | 26 ++++++++++++++- api/services/platform_importers/base.py | 29 +++++++++++++++++ api/services/platform_importers/github.py | 23 +++++++++++-- .../platform_importers/huggingface.py | 11 +++++-- api/services/platform_importers/kaggle.py | 32 +++++++++++++------ 5 files changed, 105 insertions(+), 16 deletions(-) diff --git a/api/services/platform_import_service.py b/api/services/platform_import_service.py index e60e2304..ffcaf5d8 100644 --- a/api/services/platform_import_service.py +++ b/api/services/platform_import_service.py @@ -153,7 +153,6 @@ def _prefill_metadata(dataset: Dataset, info: PlatformDatasetInfo) -> None: logger.info("platform_import_metadata_skipped", label=field.label, error=str(exc)) -@transaction.atomic def import_platform_dataset( *, platform: str, @@ -172,9 +171,34 @@ def import_platform_dataset( if existing is not None: raise DuplicateImportError(existing) + # Network first, database second: the platform calls (up to a few seconds) + # happen before any transaction is opened, so a slow platform never holds a + # database connection. if info is None: info = importer.fetch_dataset_info(canonical_id) + # The platform may have canonicalised the id (Hugging Face "imdb" -> + # "stanfordnlp/imdb"); re-check duplicates under the canonical id too. + if info.identifier != canonical_id: + existing = find_existing_import(platform, info.identifier, organization, user) + if existing is not None: + raise DuplicateImportError(existing) + + with transaction.atomic(): + return _create_import( + info, user=user, organization=organization, dataspace=dataspace, title=title + ) + + +def _create_import( + info: PlatformDatasetInfo, + *, + user: User, + organization: Optional[Organization], + dataspace: Optional[DataSpace], + title: Optional[str], +) -> Dataset: + """All database writes for one import. Runs inside a transaction.""" # Publisher may choose the name shown on DataSpace; platform title otherwise. display_title = (title or "").strip()[:300] or info.title dataset = Dataset.objects.create( diff --git a/api/services/platform_importers/base.py b/api/services/platform_importers/base.py index 3efcd360..61518bfb 100644 --- a/api/services/platform_importers/base.py +++ b/api/services/platform_importers/base.py @@ -8,6 +8,7 @@ from __future__ import annotations +import re from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime @@ -121,6 +122,30 @@ def map_license(raw: str) -> str: return LICENSE_ALIASES.get(key, DatasetLicense.CC_BY_4_0_ATTRIBUTION) +# Hard limits: the platforms' own maxima are well under these, so anything +# longer is not a real identifier. Keeps hostile input out of URLs and columns. +MAX_IDENTIFIER_LEN = 200 +MAX_README_CHARS = 200_000 # ~200 KB; the longest real card seen is ~32 KB + +_SEGMENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + + +def check_identifier_length(value: str, platform_label: str) -> None: + if len(value) > MAX_IDENTIFIER_LEN: + raise InvalidIdentifierError(f"That {platform_label} identifier is too long to be real") + + +def check_path_segments(path: str, what: str) -> None: + """A branch name or sub-path must be plain segments: no '..', no whitespace, no odd characters.""" + for seg in (path or "").split("/"): + if seg and (seg in (".", "..") or not _SEGMENT_RE.match(seg)): + raise InvalidIdentifierError(f"'{path}' is not a valid {what}") + + +def cap_readme(text: str) -> str: + return (text or "")[:MAX_README_CHARS] + + def field_type_for(dtype: Any) -> str: """Map a platform column type (Hugging Face / Arrow style names) onto FieldTypes.""" name = str(dtype if isinstance(dtype, str) else "").lower() @@ -234,6 +259,10 @@ def _get_json(self, url: str, **kwargs: Any) -> Any: "PlatformAuthError", "PlatformUnavailableError", "map_license", + "check_identifier_length", + "check_path_segments", + "cap_readme", + "MAX_IDENTIFIER_LEN", "field_type_for", "shorten", "parse_iso_datetime", diff --git a/api/services/platform_importers/github.py b/api/services/platform_importers/github.py index 8a525228..e5792b18 100644 --- a/api/services/platform_importers/github.py +++ b/api/services/platform_importers/github.py @@ -20,6 +20,9 @@ PlatformDatasetInfo, PlatformImporter, PlatformImportError, + cap_readme, + check_identifier_length, + check_path_segments, parse_iso_datetime, shorten, ) @@ -94,7 +97,13 @@ def _parse(self, value: str) -> Tuple[str, str, Optional[str], str]: raise InvalidIdentifierError( "GitHub owner and repo names use letters, digits, '-', '_' and '.'" ) - return owner, repo, (branch or None), path.strip("/") + path = path.strip("/") + if branch: + check_path_segments(branch, "branch name") + if path: + check_path_segments(path, "folder path") + check_identifier_length(f"{owner}/{repo}@{branch or ''}:{path}", "GitHub") + return owner, repo, (branch or None), path # -- fetch --------------------------------------------------------------- # def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: @@ -126,12 +135,12 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: description=shorten(readme, MAX_DESCRIPTION), source_url=source_url, author=str((meta.get("owner") or {}).get("login") or owner)[:300], - license=str(license_info.get("spdx_id") or license_info.get("name") or ""), + license=self._license(license_info), tags=[t[:50] for t in (meta.get("topics") or []) if isinstance(t, str)][:30], last_updated=parse_iso_datetime(meta.get("pushed_at") or meta.get("updated_at")), created_at=parse_iso_datetime(meta.get("created_at")), revision=self._head_sha(full_name, branch, headers), - readme=readme, + readme=cap_readme(readme), homepage=str(meta.get("homepage") or "")[:500], is_archived=bool(meta.get("archived")), ) @@ -156,3 +165,11 @@ def _head_sha(self, full_name: str, branch: str, headers: Dict[str, str]) -> str return str((ref.get("object") or {}).get("sha") or "")[:64] except PlatformImportError: return "" + + @staticmethod + def _license(license_info: Dict[str, Any]) -> str: + """SPDX id when GitHub could detect one. 'NOASSERTION' / 'other' mean it could not.""" + spdx = str(license_info.get("spdx_id") or "") + if spdx.upper() in ("", "NOASSERTION", "OTHER"): + return "" + return spdx diff --git a/api/services/platform_importers/huggingface.py b/api/services/platform_importers/huggingface.py index bf4ea994..8a8b4c20 100644 --- a/api/services/platform_importers/huggingface.py +++ b/api/services/platform_importers/huggingface.py @@ -21,6 +21,8 @@ PlatformDatasetInfo, PlatformImporter, PlatformImportError, + cap_readme, + check_identifier_length, field_type_for, parse_iso_datetime, shorten, @@ -97,6 +99,7 @@ def parse_identifier(self, value: str) -> str: repo_id = raw[len("datasets/") :] if raw.startswith("datasets/") else raw repo_id = repo_id.strip("/") + check_identifier_length(repo_id, "Hugging Face") if not _ID_RE.match(repo_id): raise InvalidIdentifierError( "Hugging Face ids look like 'namespace/name' (letters, digits, '-', '_', '.')" @@ -125,6 +128,10 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: if meta.get("disabled"): raise PlatformImportError("This dataset has been disabled on Hugging Face") + # The Hub redirects legacy short names ("imdb") to their canonical id; keep + # the canonical one so the same dataset cannot be imported twice under two names. + repo_id = str(meta.get("id") or repo_id) + card: Dict[str, Any] = meta.get("cardData") or {} tags: List[str] = meta.get("tags") or [] @@ -144,8 +151,8 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: last_updated=parse_iso_datetime(meta.get("lastModified")), created_at=parse_iso_datetime(meta.get("createdAt")), revision=str(meta.get("sha") or "")[:64], - readme=readme, - citation=str(meta.get("citation") or ""), + readme=cap_readme(readme), + citation=str(meta.get("citation") or "")[:20_000], languages=self._languages(card, tags), columns=self._columns(card), ) diff --git a/api/services/platform_importers/kaggle.py b/api/services/platform_importers/kaggle.py index bb831254..45bbba44 100644 --- a/api/services/platform_importers/kaggle.py +++ b/api/services/platform_importers/kaggle.py @@ -20,6 +20,8 @@ PlatformAuthError, PlatformDatasetInfo, PlatformImporter, + cap_readme, + check_identifier_length, parse_iso_datetime, shorten, ) @@ -68,6 +70,7 @@ def parse_identifier(self, value: str) -> str: raise InvalidIdentifierError("Kaggle refs look like 'owner/dataset-name'") owner, slug = parts + check_identifier_length(f"{owner}/{slug}", "Kaggle") if not (_SLUG_RE.match(owner) and _SLUG_RE.match(slug)): raise InvalidIdentifierError( "Kaggle owner and dataset names use letters, digits, '-' and '_'" @@ -79,9 +82,16 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: ref = self.parse_identifier(identifier) owner, slug = ref.split("/") - meta: Dict[str, Any] = self._get_json( - f"{KAGGLE_API}/datasets/view/{owner}/{slug}", auth=self._auth() - ) + try: + meta: Dict[str, Any] = self._get_json( + f"{KAGGLE_API}/datasets/view/{owner}/{slug}", auth=self._auth() + ) + except PlatformAuthError as exc: + # Kaggle answers 403 for datasets that do not exist as well as private ones. + raise PlatformAuthError( + f"'{ref}' was not found on Kaggle, or it is private. " + "Check the spelling; only public datasets can be imported." + ) from exc if meta.get("isPrivate"): raise PlatformAuthError("This Kaggle dataset is private") @@ -101,7 +111,7 @@ def fetch_dataset_info(self, identifier: str) -> PlatformDatasetInfo: last_updated=parse_iso_datetime(meta.get("lastUpdated")), created_at=self._first_version_date(meta), revision=str(version) if version is not None else "", - readme=readme, + readme=cap_readme(readme), ) # -- pieces -------------------------------------------------------------- # @@ -126,11 +136,13 @@ def _tags(meta: Dict[str, Any]) -> List[str]: @staticmethod def _first_version_date(meta: Dict[str, Any]): - """Kaggle has no created date; the earliest version's creation date is the same thing.""" - dates = [ - parse_iso_datetime(v.get("creationDate")) - for v in (meta.get("versions") or []) - if isinstance(v, dict) - ] + """Kaggle has no created date. The earliest version's date is the same thing, + but the view only lists recent versions for datasets with many, so use it + only when the list is complete.""" + versions = [v for v in (meta.get("versions") or []) if isinstance(v, dict)] + current = meta.get("currentVersionNumber") + if not versions or (isinstance(current, int) and len(versions) < current): + return None + dates = [parse_iso_datetime(v.get("creationDate")) for v in versions] dates = [d for d in dates if d is not None] return min(dates) if dates else None