feat: implement rubygems registry [CM-1241]

scripts/builders/packages.env

@@ -1,4 +1,4 @@
 DOCKERFILE="./services/docker/Dockerfile.packages"
 CONTEXT="../"
 REPO="sjc.ocir.io/axbydjxa5zuh/packages"
-SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker"
+SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker rubygems-worker"

scripts/services/rubygems-worker.yaml

@@ -0,0 +1,65 @@
+version: '3.1'
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: rubygems-worker
+  SHELL: /bin/sh
+  SUPPRESS_NO_CONFIG_WARNING: 'true'
+  CROWD_TEMPORAL_TASKQUEUE: rubygems-worker
+
+services:
+  rubygems-worker:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run start:rubygems-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    restart: always
+    networks:
+      - crowd-bridge
+
+  rubygems-worker-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run dev:rubygems-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: rubygems-worker
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
+      - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
+      - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
+      - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
+      - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
+      - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
+      - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
+      - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
+      - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
+      - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
+      - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
+      - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
+      - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
+      - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
+      - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
+      - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
+
+networks:
+  crowd-bridge:
+    external: true

services/apps/packages_worker/package.json

@@ -49,6 +49,9 @@
     "start:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker tsx src/bin/nuget-worker.ts",
     "dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
     "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
+    "start:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker tsx src/bin/rubygems-worker.ts",
+    "dev:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts",
+    "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
     "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",

services/apps/packages_worker/src/activities.ts

@@ -32,6 +32,11 @@ export {
 } from './pypi/activities'
 export { getCriticalPypiCount } from './pypi/downloads/getCriticalPypiCount'
 export { processNuGetBatch } from './nuget/activities'
+export {
+  processRubyGemsCoreBatch,
+  processRubyGemsCriticalBatch,
+  processRubyGemsDependentsBatch,
+} from './rubygems/activities'
 export {
   processSecurityContactsBatch,
   ingestSecurityContactsForPurlActivity,

services/apps/packages_worker/src/bin/rubygems-worker.ts

@@ -0,0 +1,14 @@
+import {
+  scheduleRubyGemsCriticalIngestion,
+  scheduleRubyGemsDependentsIngestion,
+  scheduleRubyGemsIngestion,
+} from '../rubygems/schedule'
+import { svc } from '../service'
+
+setImmediate(async () => {
+  await svc.init()
+  await scheduleRubyGemsIngestion()
+  await scheduleRubyGemsCriticalIngestion()
+  await scheduleRubyGemsDependentsIngestion()
+  await svc.start()
+})

services/apps/packages_worker/src/config.ts

@@ -86,6 +86,29 @@ export function getNuGetConfig() {
   }
 }
 
+export function getRubyGemsConfig() {
+  return {
+    batchSize: parseInt(process.env.RUBYGEMS_FETCHER_BATCH_SIZE ?? '1000', 10),
+    concurrency: parseInt(process.env.RUBYGEMS_FETCHER_CONCURRENCY ?? '8', 10),
+    groupDelayMs: parseInt(process.env.RUBYGEMS_FETCHER_GROUP_DELAY_MS ?? '1000', 10),
+  }
+}
+
+export function getRubyGemsCriticalConfig() {
+  return {
+    batchSize: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_BATCH_SIZE ?? '5000', 10),
+    concurrency: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_CONCURRENCY ?? '4', 10),
+    groupDelayMs: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_GROUP_DELAY_MS ?? '1000', 10),
+  }
+}
+
+export function getRubyGemsDependentsConfig() {
+  return {
+    batchSize: parseInt(process.env.RUBYGEMS_DEPENDENTS_BATCH_SIZE ?? '50000', 10),
+    concurrency: parseInt(process.env.RUBYGEMS_DEPENDENTS_CONCURRENCY ?? '50', 10),
+  }
+}
+
 export function getDockerhubConfig() {
   return {
     hubBaseUrl: requireEnv('DOCKERHUB_API_BASE_URL'),

services/apps/packages_worker/src/rubygems/activities.ts

@@ -0,0 +1,43 @@
+import { getServiceChildLogger } from '@crowd/logging'
+
+import {
+  getRubyGemsConfig,
+  getRubyGemsCriticalConfig,
+  getRubyGemsDependentsConfig,
+} from '../config'
+import { getPackagesDb } from '../db'
+
+import { processBatch as processCoreBatch } from './runRubyGemsCoreLoop'
+import { processBatch as processCriticalBatch } from './runRubyGemsCriticalLoop'
+import {
+  DependentsBatchResult,
+  processBatch as processDependentsBatch,
+} from './runRubyGemsDependentsLoop'
+import { BatchResult } from './types'
+
+const log = getServiceChildLogger('rubygems-activity')
+
+export async function processRubyGemsCoreBatch(): Promise<BatchResult> {
+  const config = getRubyGemsConfig()
+  const qx = await getPackagesDb()
+  const today = new Date().toISOString().split('T')[0]
+  const result = await processCoreBatch(qx, config, today)
+  log.info({ ...result }, 'RubyGems core batch complete')
+  return result
+}
+
+export async function processRubyGemsCriticalBatch(): Promise<BatchResult> {
+  const config = getRubyGemsCriticalConfig()
+  const qx = await getPackagesDb()
+  const result = await processCriticalBatch(qx, config)
+  log.info({ ...result }, 'RubyGems critical batch complete')
+  return result
+}
+
+export async function processRubyGemsDependentsBatch(): Promise<DependentsBatchResult> {
+  const config = getRubyGemsDependentsConfig()
+  const qx = await getPackagesDb()
+  const result = await processDependentsBatch(qx, config)
+  log.info({ ...result }, 'RubyGems dependents batch complete')
+  return result
+}

services/apps/packages_worker/src/rubygems/client.ts

@@ -0,0 +1,77 @@
+import axios from 'axios'
+
+import {
+  RubyGemsFetchError,
+  RubyGemsFetchResult,
+  RubyGemsGemResponse,
+  RubyGemsOwner,
+  RubyGemsVersionItem,
+} from './types'
+
+function classifyError(err: unknown): RubyGemsFetchError | null {
+  if (!axios.isAxiosError(err)) return null
+  const status = err.response?.status
+  if (status === 404) return { kind: 'NOT_FOUND', status, message: err.message }
+  if (status === 429) return { kind: 'RATE_LIMIT', status, message: err.message }
+  return null
+}
+
+export async function fetchGem(name: string): Promise<RubyGemsFetchResult<RubyGemsGemResponse>> {
+  try {
+    const resp = await axios.get<RubyGemsGemResponse>(
+      `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}.json`,
+      { timeout: 15000 },
+    )
+    return resp.data
+  } catch (err) {
+    const classified = classifyError(err)
+    if (classified) return classified
+    throw err
+  }
+}
+
+export async function fetchVersions(
+  name: string,
+): Promise<RubyGemsFetchResult<RubyGemsVersionItem[]>> {
+  try {
+    const resp = await axios.get<RubyGemsVersionItem[]>(
+      `https://rubygems.org/api/v1/versions/${encodeURIComponent(name)}.json`,
+      { timeout: 15000 },
+    )
+    return resp.data
+  } catch (err) {
+    const classified = classifyError(err)
+    if (classified) return classified
+    throw err
+  }
+}
+
+export async function fetchOwners(name: string): Promise<RubyGemsFetchResult<RubyGemsOwner[]>> {
+  try {
+    const resp = await axios.get<RubyGemsOwner[]>(
+      `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/owners.json`,
+      { timeout: 15000 },
+    )
+    return resp.data
+  } catch (err) {
+    const classified = classifyError(err)
+    if (classified) return classified
+    throw err
+  }
+}
+
+export async function fetchReverseDependencies(
+  name: string,
+): Promise<RubyGemsFetchResult<string[]>> {
+  try {
+    const resp = await axios.get<string[]>(
+      `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/reverse_dependencies.json`,
+      { timeout: 15000 },
+    )
+    return resp.data
+  } catch (err) {
+    const classified = classifyError(err)
+    if (classified) return classified
+    throw err
+  }
+}

services/apps/packages_worker/src/rubygems/normalize.ts

@@ -0,0 +1,63 @@
+import {
+  NormalizedRubyGemsOwner,
+  NormalizedRubyGemsPackage,
+  NormalizedRubyGemsVersion,
+  RubyGemsGemResponse,
+  RubyGemsOwner,
+  RubyGemsVersionItem,
+} from './types'
+
+function nonEmpty(value: string | null | undefined): string | null {
+  if (!value) return null
+  const trimmed = value.trim()
+  return trimmed === '' ? null : trimmed
+}
+
+export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRubyGemsPackage {
+  const licenses = doc.licenses && doc.licenses.length > 0 ? doc.licenses : null
+  return {
+    description: nonEmpty(doc.info),
+    homepage: nonEmpty(doc.homepage_uri),
+    declaredRepositoryUrl: nonEmpty(doc.source_code_uri),
+    licenses,
+    licensesRaw: licenses ? licenses.join(', ') : null,
+    latestVersion: nonEmpty(doc.version),
+    totalDownloads: doc.downloads ?? 0,
+  }
+}
+
+function parseCreatedAt(value: string | undefined): Date | null {
+  if (!value) return null
+  const date = new Date(value)
+  return isNaN(date.getTime()) ? null : date
+}
+
+export function normalizeRubyGemsVersions(
+  items: RubyGemsVersionItem[],
+): NormalizedRubyGemsVersion[] {
+  return items.map((item) => ({
+    number: item.number,
+    publishedAt: parseCreatedAt(item.created_at),
+    isPrerelease: item.prerelease ?? false,
+    licenses: item.licenses && item.licenses.length > 0 ? item.licenses : null,
+  }))
+}
+
+export function pickLatestRubyGemsVersion(
+  versions: NormalizedRubyGemsVersion[],
+): NormalizedRubyGemsVersion | null {
+  if (versions.length === 0) return null
+  const sorted = [...versions].sort((a, b) => {
+    const at = a.publishedAt ? a.publishedAt.getTime() : 0
+    const bt = b.publishedAt ? b.publishedAt.getTime() : 0
+    return bt - at
+  })
+  const stable = sorted.find((v) => !v.isPrerelease)
+  return stable ?? sorted[0]
+}
+
+export function normalizeRubyGemsOwners(owners: RubyGemsOwner[]): NormalizedRubyGemsOwner[] {
+  return owners
+    .filter((o): o is RubyGemsOwner & { handle: string } => !!o.handle && o.handle.trim() !== '')
+    .map((o) => ({ username: o.handle, email: nonEmpty(o.email) }))
+}