From d19e6010a9a93588b2d15c80718db18fb8c6fb89 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Sat, 8 Aug 2026 01:00:32 -0400 Subject: [PATCH 01/23] refactor: modularize GitHub metadata import pipeline Split importer responsibilities into focused modules for author matching, contributor retrieval, file parsing, and shared normalization utilities. Centralized author and repository normalization logic to reduce duplication and improve maintainability. Updated citation validation tests to verify required author-field handling remains correct after the refactor. --- src/services/githubImporter.js | 981 ++------------------- src/services/githubImporterAuthors.js | 269 ++++++ src/services/githubImporterContributors.js | 258 ++++++ src/services/githubImporterParsers.js | 170 ++++ src/services/githubImporterUtils.js | 271 ++++++ tests/services/citationValidation.test.js | 10 +- 6 files changed, 1029 insertions(+), 930 deletions(-) create mode 100644 src/services/githubImporterAuthors.js create mode 100644 src/services/githubImporterContributors.js create mode 100644 src/services/githubImporterParsers.js create mode 100644 src/services/githubImporterUtils.js diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 0822a3d..e2b85f3 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -1,5 +1,5 @@ import { createMetadata } from '../core/metadataModel.js'; -import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile, normalizeOrcid } from '../utils/orcid.js'; +import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile } from '../utils/orcid.js'; import { validateCitationCffText } from './citationValidation.js'; import { runCitationHealthScan } from './citationHealthScan.js'; import { @@ -10,14 +10,40 @@ import { parseGithubUrl, resolveGithubToken, } from './githubApi.js'; +import { + cleanString as utilCleanString, + extractFirstMarkdownParagraph as utilExtractFirstMarkdownParagraph, + firstNonEmpty as utilFirstNonEmpty, + normalizeAuthor as utilNormalizeAuthor, + normalizeAuthors as utilNormalizeAuthors, + normalizeGrants as utilNormalizeGrants, + normalizeKeywords as utilNormalizeKeywords, + normalizeReferences as utilNormalizeReferences, + normalizeRepoUrl as utilNormalizeRepoUrl, + normalizeVersionForCompare as utilNormalizeVersionForCompare, +} from './githubImporterUtils.js'; +import { + fetchContributorAuthors, + resolveContributorFallbackLimit, +} from './githubImporterContributors.js'; +import { + dedupeAuthors, + enrichAuthorsWithContributorData, + orderAuthorsByContributorRank, +} from './githubImporterAuthors.js'; +import { + parseCargoToml, + parsePackageJson, + parsePomXml, + parsePyprojectToml, + parseReadme, + parseSetupPy, +} from './githubImporterParsers.js'; import { compareExistingMetadataFiles } from './metadataComparison.js'; import { runMetadataReviewPipeline } from './metadataReview.js'; import { validateZenodoJsonText } from './zenodoValidation.js'; const API_BASE = 'https://api.github.com'; -const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; -const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; -const GITHUB_PAGE_SIZE = 100; const FILES_TO_INSPECT = [ 'CITATION.cff', '.zenodo.json', @@ -29,6 +55,17 @@ const FILES_TO_INSPECT = [ 'pom.xml', ]; +const cleanString = utilCleanString; +const firstNonEmpty = utilFirstNonEmpty; +const normalizeKeywords = utilNormalizeKeywords; +const normalizeReferences = utilNormalizeReferences; +const normalizeGrants = utilNormalizeGrants; +const normalizeAuthor = utilNormalizeAuthor; +const normalizeAuthors = utilNormalizeAuthors; +const normalizeRepoUrl = utilNormalizeRepoUrl; +const normalizeVersionForCompare = utilNormalizeVersionForCompare; +const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; + function makeIssue(kind, source, code, message, details = {}) { return { kind, source, code, message, ...details }; } @@ -41,234 +78,6 @@ function addError(errors, source, code, message, details = {}) { errors.push(makeIssue('error', source, code, message, details)); } -function cleanString(value) { - return String(value ?? '').replace(/[\t ]+/g, ' ').trim(); -} - -function firstNonEmpty(...values) { - for (const value of values) { - if (Array.isArray(value)) { - if (value.length > 0) { - return value; - } - continue; - } - - const text = cleanString(value); - if (text) { - return text; - } - } - - return ''; -} - -function normalizeStringList(value) { - if (Array.isArray(value)) { - return value.map((item) => cleanString(item)).filter(Boolean); - } - - if (!value) { - return []; - } - - return String(value) - .split(/[\n,]/) - .map((item) => cleanString(item)) - .filter(Boolean); -} - -function normalizeKeywords(value) { - return [...new Set(normalizeStringList(value).map((keyword) => keyword.toLowerCase()))]; -} - -function normalizeReferences(value) { - if (Array.isArray(value)) { - return value.map((item) => cleanString(item)).filter(Boolean); - } - - if (!value) { - return []; - } - - return String(value) - .split(/\n+/) - .map((item) => cleanString(item)) - .filter(Boolean); -} - -function normalizeGrants(value) { - if (Array.isArray(value)) { - return value - .map((item) => { - if (typeof item === 'string') { - return cleanString(item); - } - - if (item && typeof item === 'object') { - return cleanString(item.id ?? item.value ?? item.grantId ?? ''); - } - - return ''; - }) - .filter(Boolean); - } - - if (!value) { - return []; - } - - return String(value) - .split(/\n+/) - .map((item) => cleanString(item)) - .filter(Boolean); -} - -function capitalizeToken(token) { - const text = cleanString(token); - if (!text) { - return ''; - } - - return text - .split(/([\-'])/) - .map((part) => { - if (part === '-' || part === "'") { - return part; - } - - // Preserve mixed-case tokens (for example, McDonald) and normalize others. - if (/[a-z]/.test(part) && /[A-Z]/.test(part)) { - return part; - } - - const lower = part.toLowerCase(); - return lower.charAt(0).toUpperCase() + lower.slice(1); - }) - .join(''); -} - -function capitalizeName(value) { - return cleanString(value) - .split(/\s+/) - .map((part) => capitalizeToken(part)) - .filter(Boolean) - .join(' '); -} - -function humanizeIdentifier(value) { - return cleanString(value) - .replace(/[._-]+/g, ' ') - .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') - .replace(/([a-z\d])([A-Z])/g, '$1 $2') - .replace(/([A-Za-z])(\d)/g, '$1 $2') - .replace(/(\d)([A-Za-z])/g, '$1 $2'); -} - -function splitDisplayName(name) { - const value = cleanString(name); - - if (!value) { - return { givenNames: '', familyNames: '' }; - } - - if (value.includes(',')) { - const [familyNames, ...givenParts] = value.split(','); - return { - givenNames: capitalizeName(givenParts.join(',').trim()), - familyNames: capitalizeName(familyNames), - }; - } - - const normalized = humanizeIdentifier(value); - const parts = normalized.split(/\s+/).filter(Boolean); - - if (parts.length <= 1) { - return { givenNames: capitalizeName(parts[0] ?? ''), familyNames: '' }; - } - - return { - givenNames: capitalizeName(parts.slice(0, -1).join(' ')), - familyNames: capitalizeName(parts[parts.length - 1]), - }; -} - -function normalizeAuthor(input) { - if (!input) { - return null; - } - - if (typeof input === 'string') { - const { givenNames, familyNames } = splitDisplayName(input); - return givenNames || familyNames ? { givenNames, familyNames, orcid: '', affiliation: '' } : null; - } - - if (typeof input !== 'object') { - return null; - } - - const name = cleanString(input.name ?? input.fullName ?? input.full_name ?? input.creator_name ?? ''); - const parsedName = name ? splitDisplayName(name) : null; - let givenNames = capitalizeName(input.givenNames ?? input['given-names'] ?? input.firstName ?? input.firstname ?? parsedName?.givenNames ?? ''); - let familyNames = capitalizeName(input.familyNames ?? input['family-names'] ?? input.lastName ?? input.lastname ?? parsedName?.familyNames ?? ''); - const affiliation = cleanString(input.affiliation ?? input.organization ?? input.company ?? input.institution ?? ''); - const orcid = normalizeOrcid(input.orcid ?? input.ORCID ?? input.orcidId ?? ''); - - // Some sources put full names in a single first-name field without spaces. - if (givenNames && !familyNames) { - const reparsed = splitDisplayName(givenNames); - if (reparsed.familyNames) { - givenNames = reparsed.givenNames; - familyNames = reparsed.familyNames; - } - } - - if (!givenNames && !familyNames && !affiliation && !orcid) { - return null; - } - - return { givenNames, familyNames, orcid, affiliation }; -} - -function normalizeAuthors(value) { - if (!Array.isArray(value)) { - return []; - } - - return value.map((item) => normalizeAuthor(item)).filter(Boolean); -} - -function normalizeRepoUrl(value) { - const text = cleanString(value); - if (!text) { - return ''; - } - - const trimmed = text.replace(/^git\+/, '').replace(/\.git$/i, '').replace(/\/+$/, ''); - - try { - const parsed = new URL(trimmed); - const host = parsed.hostname.toLowerCase(); - let pathname = parsed.pathname.replace(/\/+$/, ''); - if (host === 'github.com') { - pathname = pathname.toLowerCase(); - } - return `${parsed.protocol}//${host}${pathname}`; - } catch { - return trimmed; - } -} - -function normalizeVersionForCompare(value) { - const text = cleanString(value).toLowerCase(); - if (!text) { - return ''; - } - - // Treat v-prefixed tags and bare semver as equivalent for mismatch checks. - return text.replace(/^v(?=\d)/, ''); -} - function addRateLimitHintIfNeeded(warnings, authToken) { if (authToken) { return; @@ -287,30 +96,6 @@ function addRateLimitHintIfNeeded(warnings, authToken) { } } -async function fetchOrcidFromGithubProfileHtml(profileUrl) { - const url = cleanString(profileUrl); - if (!url) { - return null; - } - - try { - const response = await fetch(url, { - headers: { - Accept: 'text/html', - }, - }); - - if (!response.ok) { - return null; - } - - const html = await response.text(); - return extractOrcidFromGithubHtml(html); - } catch { - return null; - } -} - function shouldInspectRepositoryFiles(options = {}) { return options.inspectRepositoryFiles !== false; } @@ -394,90 +179,6 @@ export function summarizeImportedMetadataFiles(fileContents = {}) { return summary; } -function resolveContributorFallbackLimit(options = {}) { - if (!Object.prototype.hasOwnProperty.call(options, 'contributorFallbackLimit')) { - return TOP_CONTRIBUTOR_FALLBACK_LIMIT; - } - - if (options.contributorFallbackLimit == null || options.contributorFallbackLimit === '') { - return null; - } - - const rawLimit = Number(options.contributorFallbackLimit); - - if (!Number.isFinite(rawLimit)) { - return TOP_CONTRIBUTOR_FALLBACK_LIMIT; - } - - return Math.min(Math.max(Math.trunc(rawLimit), 1), MAX_CONTRIBUTOR_FALLBACK_LIMIT); -} - -async function fetchAllContributors(owner, repo, warnings, authToken = '', maxContributors = null) { - const contributors = []; - let page = 1; - - while (true) { - const pageContributors = await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, - { - authToken, - source: 'contributors', - label: `contributors page ${page}`, - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, - ) || []; - - if (!Array.isArray(pageContributors) || pageContributors.length === 0) { - break; - } - - contributors.push(...pageContributors); - - if (maxContributors && contributors.length >= maxContributors) { - return contributors.slice(0, maxContributors); - } - - if (pageContributors.length < GITHUB_PAGE_SIZE) { - break; - } - - page += 1; - } - - return contributors; -} - -function extractFirstMarkdownParagraph(text) { - const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); - const paragraph = []; - let started = false; - - for (const line of lines) { - const trimmed = line.trim(); - - if (!trimmed) { - if (started) { - break; - } - continue; - } - - if (!started && /^#{1,6}\s+/.test(trimmed)) { - started = true; - continue; - } - - if (!started && /^(!|\[|-)/.test(trimmed)) { - continue; - } - - started = true; - paragraph.push(trimmed); - } - - return paragraph.join(' ').replace(/\s+/g, ' ').trim(); -} - export function parseCitationCff(text) { const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); const result = { @@ -763,159 +464,6 @@ export function parseZenodoJson(text) { }; } -function extractPackageAuthors(payload) { - const candidates = []; - - if (payload.author) { - candidates.push(payload.author); - } - - if (Array.isArray(payload.authors)) { - candidates.push(...payload.authors); - } - - return candidates.map((item) => normalizeAuthor(item)).filter(Boolean); -} - -function parsePackageJson(text) { - const payload = parseJsonSafely(text); - - return { - title: cleanString(payload.name ?? ''), - abstract: cleanString(payload.description ?? ''), - version: cleanString(payload.version ?? ''), - repositoryCode: normalizeRepoUrl(typeof payload.repository === 'string' ? payload.repository : payload.repository?.url ?? payload.homepage ?? ''), - license: cleanString(typeof payload.license === 'string' ? payload.license : payload.license?.type ?? ''), - keywords: normalizeKeywords(payload.keywords), - authors: extractPackageAuthors(payload), - }; -} - -function extractTomlSection(text, sectionName) { - const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); - const sectionLines = []; - let inSection = false; - - for (const line of lines) { - const sectionMatch = line.trim().match(/^\[([^\]]+)\]$/); - - if (sectionMatch) { - if (inSection) { - break; - } - - inSection = sectionMatch[1] === sectionName; - continue; - } - - if (inSection) { - sectionLines.push(line); - } - } - - return sectionLines.join('\n'); -} - -function extractTomlValue(sectionText, key) { - const match = sectionText.match(new RegExp(`^${key}\\s*=\\s*(.+)$`, 'm')); - return match ? match[1].trim() : ''; -} - -function parseTomlString(sectionText, key) { - const value = extractTomlValue(sectionText, key); - const match = value.match(/^['"](.+?)['"]$/); - return match ? match[1].trim() : ''; -} - -function parseTomlStrings(value) { - return [...String(value ?? '').matchAll(/['"]([^'"]+)['"]/g)].map((match) => cleanString(match[1])).filter(Boolean); -} - -function parsePyprojectToml(text) { - const section = extractTomlSection(text, 'project') || extractTomlSection(text, 'tool.poetry'); - const authorsBlock = extractTomlValue(section, 'authors') || extractTomlValue(section, 'maintainers'); - const authors = [...String(authorsBlock ?? '').matchAll(/name\s*=\s*['"]([^'"]+)['"]/g)] - .map((match) => normalizeAuthor({ name: match[1] })) - .filter(Boolean); - - const licenseValue = parseTomlString(section, 'license') || cleanString((section.match(/license\s*=\s*\{[^}]*text\s*=\s*['"]([^'"]+)['"][^}]*\}/s) || [])[1] ?? ''); - const repositoryCode = parseTomlString(section, 'repository') || parseTomlString(section, 'homepage') || parseTomlString(section, 'url'); - - return { - title: parseTomlString(section, 'name'), - abstract: parseTomlString(section, 'description'), - version: parseTomlString(section, 'version'), - repositoryCode, - license: licenseValue, - keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), - authors, - }; -} - -function parseSetupPy(text) { - const source = String(text ?? ''); - const extract = (key) => cleanString((source.match(new RegExp(`${key}\\s*=\\s*['"]([^'"]+)['"]`, 'm')) || [])[1] ?? ''); - - const authors = []; - const author = extract('author'); - const maintainer = extract('maintainer'); - - if (author) { - authors.push(normalizeAuthor({ name: author })); - } else if (maintainer) { - authors.push(normalizeAuthor({ name: maintainer })); - } - - return { - title: extract('name'), - abstract: extract('description'), - version: extract('version'), - repositoryCode: normalizeRepoUrl(extract('url')), - license: extract('license'), - keywords: normalizeKeywords(extract('keywords')), - authors: authors.filter(Boolean), - }; -} - -function parseCargoToml(text) { - const section = extractTomlSection(text, 'package'); - const authors = parseTomlStrings(extractTomlValue(section, 'authors')).map((name) => normalizeAuthor({ name })).filter(Boolean); - - return { - title: parseTomlString(section, 'name'), - abstract: parseTomlString(section, 'description'), - version: parseTomlString(section, 'version'), - repositoryCode: parseTomlString(section, 'repository'), - license: parseTomlString(section, 'license'), - keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), - authors, - }; -} - -function parsePomXml(text) { - const source = String(text ?? ''); - const extract = (pattern) => cleanString((source.match(pattern) || [])[1] ?? ''); - const authors = [...source.matchAll(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/developer>/g)] - .map((match) => normalizeAuthor({ name: match[1] })) - .filter(Boolean); - - const licenseMatch = source.match(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/license>/); - - return { - title: extract(/([^<]+)<\/name>/), - abstract: extract(/([^<]+)<\/description>/), - version: extract(/([^<]+)<\/version>/), - repositoryCode: extract(/([^<]+)<\/url>/), - license: cleanString((licenseMatch || [])[1] ?? ''), - keywords: [], - authors, - }; -} - -function parseReadme(text) { - return extractFirstMarkdownParagraph(text); -} - function parseFile(path, text, warnings, errors) { try { if (path === '.zenodo.json') { @@ -983,123 +531,6 @@ function mapTypeOfWork(value) { return 'software'; } -function authorNameKey(author) { - return [ - cleanString(author?.givenNames ?? '').toLowerCase(), - cleanString(author?.familyNames ?? '').toLowerCase(), - ].join('|'); -} - -function normalizeNameToken(value) { - return cleanString(value) - .toLowerCase() - .replace(/[^a-z0-9\s]/g, ' ') - .replace(/\s+/g, ' ') - .trim(); -} - -function authorMatchMetadata(author) { - const givenNames = normalizeNameToken(author?.givenNames ?? ''); - const familyNames = normalizeNameToken(author?.familyNames ?? ''); - const givenTokens = givenNames.split(' ').filter(Boolean); - const familyTokens = familyNames.split(' ').filter(Boolean); - - return { - givenNames, - familyNames, - givenFirst: givenTokens[0] ?? '', - givenInitials: givenTokens.map((token) => token[0]).join(''), - familyLast: familyTokens[familyTokens.length - 1] ?? '', - fullName: [givenNames, familyNames].filter(Boolean).join(' ').trim(), - }; -} - -function authorAltNameKeys(author) { - const metadata = authorMatchMetadata(author); - const givenNames = metadata.givenNames; - const familyNames = metadata.familyNames; - const givenFirst = metadata.givenFirst; - const familyLast = metadata.familyLast; - const keys = new Set([ - `${givenNames}|${familyNames}`, - `${givenFirst}|${familyNames}`, - `${givenNames}|${familyLast}`, - `${givenFirst}|${familyLast}`, - ]); - - keys.delete('|'); - keys.delete(''); - return [...keys].filter(Boolean); -} - -function authorsLikelyMatch(sourceAuthor, contributorAuthor) { - const source = authorMatchMetadata(sourceAuthor); - const contributor = authorMatchMetadata(contributorAuthor); - - if (!source.familyLast || !contributor.familyLast || source.familyLast !== contributor.familyLast) { - return false; - } - - if (source.givenNames && contributor.givenNames && source.givenNames === contributor.givenNames) { - return true; - } - - if (source.givenFirst && contributor.givenFirst && source.givenFirst === contributor.givenFirst) { - return true; - } - - if (source.givenInitials && contributor.givenInitials && source.givenInitials === contributor.givenInitials) { - return true; - } - - if (source.fullName && contributor.fullName && source.fullName === contributor.fullName) { - return true; - } - - return false; -} - -function enrichAuthorsWithContributorData(sourceAuthors, contributorAuthors) { - const normalizedSourceAuthors = normalizeAuthors(Array.isArray(sourceAuthors) ? sourceAuthors : []); - if (normalizedSourceAuthors.length === 0) { - return normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); - } - - const normalizedContributorAuthors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); - const contributorMatches = new Map(); - - for (const contributorAuthor of normalizedContributorAuthors) { - if (!contributorAuthor.orcid && !contributorAuthor.affiliation) { - continue; - } - - for (const key of authorAltNameKeys(contributorAuthor)) { - const existing = contributorMatches.get(key) ?? []; - existing.push(contributorAuthor); - contributorMatches.set(key, existing); - } - } - - return normalizedSourceAuthors.map((author) => { - const keyedMatches = authorAltNameKeys(author) - .flatMap((key) => contributorMatches.get(key) ?? []); - const heuristicMatches = normalizedContributorAuthors.filter((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); - const matches = [...new Set([...keyedMatches, ...heuristicMatches])]; - const uniqueOrcids = [...new Set(matches.map((match) => match.orcid).filter(Boolean))]; - const uniqueAffiliations = [...new Set(matches.map((match) => match.affiliation).filter(Boolean))]; - - if ((!author.orcid && uniqueOrcids.length > 1) || (!author.affiliation && uniqueAffiliations.length > 1)) { - return author; - } - - return { - ...author, - orcid: author.orcid || uniqueOrcids[0] || '', - affiliation: author.affiliation || uniqueAffiliations[0] || '', - }; - }); -} - function mergeMetadata({ repo, release, @@ -1384,14 +815,22 @@ export async function importGithubMetadata(repoUrl, options = {}) { const readme = parsedFiles['readme.md']?.abstract || ''; const hasPrimaryAuthors = firstNonEmpty(citation?.authors, zenodo?.authors, packageMeta?.authors, supplementalCitationAuthors).length > 0; - const contributorResult = await fetchContributorAuthors( + const contributorResult = await fetchContributorAuthors({ owner, repo, warnings, authToken, contributorFallbackLimit, - !hasPrimaryAuthors, - ); + emitFallbackWarning: !hasPrimaryAuthors, + cleanString, + normalizeAuthor, + normalizeAuthors, + dedupeAuthors, + addWarning, + fetchOptionalJson, + extractOrcidFromGithubProfile, + extractOrcidFromGithubHtml, + }); const contributors = contributorResult.fallbackAuthors.filter(Boolean); const contributorLookupAuthors = contributorResult.lookupAuthors.filter(Boolean); @@ -1471,319 +910,3 @@ export async function importGithubMetadata(repoUrl, options = {}) { return { metadata, warnings, errors, review, healthScan, comparisons }; } -async function fetchContributorAuthors( - owner, - repo, - warnings, - authToken = '', - contributorFallbackLimit = TOP_CONTRIBUTOR_FALLBACK_LIMIT, - emitFallbackWarning = true, -) { - const contributors = await fetchAllContributors(owner, repo, warnings, authToken, contributorFallbackLimit); - - if (!Array.isArray(contributors) || contributors.length === 0) { - return { - fallbackAuthors: [], - lookupAuthors: [], - }; - } - - if (emitFallbackWarning) { - addWarning( - warnings, - 'authors', - 'commit-based-fallback', - contributorFallbackLimit - ? `Using top ${contributorFallbackLimit} contributors as fallback authors.` - : 'Using contributors as fallback authors.', - { owner, repo }, - ); - } - - const profiles = await Promise.all( - contributors.map(async (contributor) => { - const login = cleanString(contributor?.login ?? ''); - if (!login) { - return { - contributor, - profile: null, - socialAccounts: [], - author: null, - autoFilledOrcid: false, - excludedAutomated: false, - }; - } - - const profile = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}`, - { - authToken, - source: 'contributor-profile', - label: `the profile for ${login}`, - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, - ); - - const socialAccounts = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, - { - authToken, - source: 'contributor-profile-links', - label: `the profile links for ${login}`, - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, - ) || []; - - if (isAutomatedContributor(contributor, profile)) { - return { - contributor, - profile, - socialAccounts, - author: null, - autoFilledOrcid: false, - excludedAutomated: true, - }; - } - - let profileOrcid = extractOrcidFromGithubProfile(profile, socialAccounts); - if (!profileOrcid) { - profileOrcid = await fetchOrcidFromGithubProfileHtml(profile?.html_url ?? contributor?.html_url ?? ''); - } - - if (profile?.name) { - return { - contributor, - profile, - socialAccounts, - author: normalizeAuthor({ - name: profile.name, - affiliation: profile.company ?? '', - orcid: profileOrcid, - }), - autoFilledOrcid: Boolean(profileOrcid), - excludedAutomated: false, - }; - } - - // Fallback to contributor login when profile name is missing. - return { - contributor, - profile, - socialAccounts, - author: normalizeAuthor({ - name: login, - affiliation: '', - orcid: profileOrcid, - }), - autoFilledOrcid: Boolean(profileOrcid), - excludedAutomated: false, - }; - }), - ); - - const excludedAutomatedCount = profiles.filter((entry) => entry?.excludedAutomated).length; - if (excludedAutomatedCount > 0) { - addWarning( - warnings, - 'authors', - 'automated-contributors-excluded', - `Excluded ${excludedAutomatedCount} automated account(s) from fallback authors.`, - { owner, repo }, - ); - } - - const autoFilledOrcidCount = profiles.filter((entry) => entry?.autoFilledOrcid).length; - if (autoFilledOrcidCount > 0) { - addWarning( - warnings, - 'authors', - 'orcid-autofilled', - `Auto-filled ORCID for ${autoFilledOrcidCount} contributor(s) from GitHub profile data.`, - { owner, repo }, - ); - } - - const fallbackAuthors = profiles - .slice(0, contributorFallbackLimit ?? profiles.length) - .map((entry) => entry?.author); - const lookupAuthors = profiles.map((entry) => entry?.author); - - return { - fallbackAuthors: dedupeAuthors(normalizeAuthors(fallbackAuthors)), - lookupAuthors: dedupeAuthors(normalizeAuthors(lookupAuthors)), - }; -} - -function dedupeAuthors(authors) { - const seen = new Set(); - const byOrcid = new Map(); - const byName = new Map(); - const deduped = []; - - for (const rawAuthor of authors) { - const author = normalizeAuthor(rawAuthor); - if (!author) { - continue; - } - - const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); - const nameKey = [ - cleanString(author?.givenNames ?? '').toLowerCase(), - cleanString(author?.familyNames ?? '').toLowerCase(), - ].join('|'); - - if (nameKey !== '|' && byName.has(nameKey)) { - const existing = byName.get(nameKey); - const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); - const canMergeByName = !existingOrcid || !orcidKey || existingOrcid === orcidKey; - - if (canMergeByName) { - if (!existing.orcid && author.orcid) { - existing.orcid = author.orcid; - } - if (!existing.affiliation && author.affiliation) { - existing.affiliation = author.affiliation; - } - if (orcidKey && !byOrcid.has(orcidKey)) { - byOrcid.set(orcidKey, existing); - } - continue; - } - } - - const likelyMatch = deduped.find((existing) => { - const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); - const hasConflictingOrcid = existingOrcid && orcidKey && existingOrcid !== orcidKey; - - if (hasConflictingOrcid) { - return false; - } - - return authorsLikelyMatch(existing, author); - }); - - if (likelyMatch) { - if (!likelyMatch.orcid && author.orcid) { - likelyMatch.orcid = author.orcid; - } - if (!likelyMatch.affiliation && author.affiliation) { - likelyMatch.affiliation = author.affiliation; - } - - const mergedOrcidKey = cleanString(likelyMatch?.orcid ?? '').toLowerCase(); - if (mergedOrcidKey && !byOrcid.has(mergedOrcidKey)) { - byOrcid.set(mergedOrcidKey, likelyMatch); - } - continue; - } - - if (orcidKey && byOrcid.has(orcidKey)) { - const existing = byOrcid.get(orcidKey); - if (!existing.affiliation && author.affiliation) { - existing.affiliation = author.affiliation; - } - continue; - } - - const key = [ - cleanString(author?.givenNames ?? '').toLowerCase(), - cleanString(author?.familyNames ?? '').toLowerCase(), - cleanString(author?.orcid ?? '').toLowerCase(), - ].join('|'); - - if (!key || seen.has(key)) { - continue; - } - - seen.add(key); - const normalizedAuthor = { ...author }; - deduped.push(normalizedAuthor); - - if (orcidKey) { - byOrcid.set(orcidKey, normalizedAuthor); - } - - if (nameKey !== '|') { - byName.set(nameKey, normalizedAuthor); - } - } - - return deduped; -} - -function orderAuthorsByContributorRank(authors, contributorAuthors) { - const normalizedAuthors = normalizeAuthors(Array.isArray(authors) ? authors : []); - const normalizedContributors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); - - if (normalizedContributors.length === 0 || normalizedAuthors.length <= 1) { - return normalizedAuthors; - } - - const contributorOrcidIndex = new Map(); - const contributorNameKeyIndex = new Map(); - - normalizedContributors.forEach((contributorAuthor, index) => { - const orcidKey = cleanString(contributorAuthor?.orcid ?? '').toLowerCase(); - if (orcidKey && !contributorOrcidIndex.has(orcidKey)) { - contributorOrcidIndex.set(orcidKey, index); - } - - for (const key of authorAltNameKeys(contributorAuthor)) { - if (!contributorNameKeyIndex.has(key)) { - contributorNameKeyIndex.set(key, index); - } - } - }); - - const ranked = normalizedAuthors.map((author, originalIndex) => { - const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); - if (orcidKey && contributorOrcidIndex.has(orcidKey)) { - return { author, originalIndex, rank: contributorOrcidIndex.get(orcidKey) }; - } - - const nameRanks = authorAltNameKeys(author) - .map((key) => contributorNameKeyIndex.get(key)) - .filter((value) => Number.isInteger(value)); - - if (nameRanks.length > 0) { - return { author, originalIndex, rank: Math.min(...nameRanks) }; - } - - const heuristicIndex = normalizedContributors.findIndex((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); - if (heuristicIndex >= 0) { - return { author, originalIndex, rank: heuristicIndex }; - } - - return { author, originalIndex, rank: Number.POSITIVE_INFINITY }; - }); - - ranked.sort((left, right) => { - if (left.rank !== right.rank) { - return left.rank - right.rank; - } - - return left.originalIndex - right.originalIndex; - }); - - return ranked.map((entry) => entry.author); -} - -function isAutomatedContributor(contributor, profile) { - const login = cleanString(profile?.login ?? contributor?.login ?? '').toLowerCase(); - const contributorType = cleanString(contributor?.type ?? '').toLowerCase(); - const profileType = cleanString(profile?.type ?? '').toLowerCase(); - - if ((contributorType && contributorType !== 'user') || (profileType && profileType !== 'user')) { - return true; - } - - if (!login) { - return false; - } - - if (login.endsWith('[bot]')) { - return true; - } - - return /(^|[-_])(github-actions|dependabot|copilot|codex|claude|swe-agent)([-_]|$)/.test(login); -} diff --git a/src/services/githubImporterAuthors.js b/src/services/githubImporterAuthors.js new file mode 100644 index 0000000..65dee34 --- /dev/null +++ b/src/services/githubImporterAuthors.js @@ -0,0 +1,269 @@ +import { + cleanString, + normalizeAuthor, + normalizeAuthors, +} from './githubImporterUtils.js'; + +function normalizeNameToken(value) { + return cleanString(value) + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function authorMatchMetadata(author) { + const givenNames = normalizeNameToken(author?.givenNames ?? ''); + const familyNames = normalizeNameToken(author?.familyNames ?? ''); + const givenTokens = givenNames.split(' ').filter(Boolean); + const familyTokens = familyNames.split(' ').filter(Boolean); + + return { + givenNames, + familyNames, + givenFirst: givenTokens[0] ?? '', + givenInitials: givenTokens.map((token) => token[0]).join(''), + familyLast: familyTokens[familyTokens.length - 1] ?? '', + fullName: [givenNames, familyNames].filter(Boolean).join(' ').trim(), + }; +} + +function authorAltNameKeys(author) { + const metadata = authorMatchMetadata(author); + const givenNames = metadata.givenNames; + const familyNames = metadata.familyNames; + const givenFirst = metadata.givenFirst; + const familyLast = metadata.familyLast; + const keys = new Set([ + `${givenNames}|${familyNames}`, + `${givenFirst}|${familyNames}`, + `${givenNames}|${familyLast}`, + `${givenFirst}|${familyLast}`, + ]); + + keys.delete('|'); + keys.delete(''); + return [...keys].filter(Boolean); +} + +export function authorsLikelyMatch(sourceAuthor, contributorAuthor) { + const source = authorMatchMetadata(sourceAuthor); + const contributor = authorMatchMetadata(contributorAuthor); + + if (!source.familyLast || !contributor.familyLast || source.familyLast !== contributor.familyLast) { + return false; + } + + if (source.givenNames && contributor.givenNames && source.givenNames === contributor.givenNames) { + return true; + } + + if (source.givenFirst && contributor.givenFirst && source.givenFirst === contributor.givenFirst) { + return true; + } + + if (source.givenInitials && contributor.givenInitials && source.givenInitials === contributor.givenInitials) { + return true; + } + + if (source.fullName && contributor.fullName && source.fullName === contributor.fullName) { + return true; + } + + return false; +} + +export function enrichAuthorsWithContributorData(sourceAuthors, contributorAuthors) { + const normalizedSourceAuthors = normalizeAuthors(Array.isArray(sourceAuthors) ? sourceAuthors : []); + if (normalizedSourceAuthors.length === 0) { + return normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); + } + + const normalizedContributorAuthors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); + const contributorMatches = new Map(); + + for (const contributorAuthor of normalizedContributorAuthors) { + if (!contributorAuthor.orcid && !contributorAuthor.affiliation) { + continue; + } + + for (const key of authorAltNameKeys(contributorAuthor)) { + const existing = contributorMatches.get(key) ?? []; + existing.push(contributorAuthor); + contributorMatches.set(key, existing); + } + } + + return normalizedSourceAuthors.map((author) => { + const keyedMatches = authorAltNameKeys(author) + .flatMap((key) => contributorMatches.get(key) ?? []); + const heuristicMatches = normalizedContributorAuthors.filter((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); + const matches = [...new Set([...keyedMatches, ...heuristicMatches])]; + const uniqueOrcids = [...new Set(matches.map((match) => match.orcid).filter(Boolean))]; + const uniqueAffiliations = [...new Set(matches.map((match) => match.affiliation).filter(Boolean))]; + + if ((!author.orcid && uniqueOrcids.length > 1) || (!author.affiliation && uniqueAffiliations.length > 1)) { + return author; + } + + return { + ...author, + orcid: author.orcid || uniqueOrcids[0] || '', + affiliation: author.affiliation || uniqueAffiliations[0] || '', + }; + }); +} + +export function dedupeAuthors(authors) { + const seen = new Set(); + const byOrcid = new Map(); + const byName = new Map(); + const deduped = []; + + for (const rawAuthor of authors) { + const author = normalizeAuthor(rawAuthor); + if (!author) { + continue; + } + + const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); + const nameKey = [ + cleanString(author?.givenNames ?? '').toLowerCase(), + cleanString(author?.familyNames ?? '').toLowerCase(), + ].join('|'); + + if (nameKey !== '|' && byName.has(nameKey)) { + const existing = byName.get(nameKey); + const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); + const canMergeByName = !existingOrcid || !orcidKey || existingOrcid === orcidKey; + + if (canMergeByName) { + if (!existing.orcid && author.orcid) { + existing.orcid = author.orcid; + } + if (!existing.affiliation && author.affiliation) { + existing.affiliation = author.affiliation; + } + if (orcidKey && !byOrcid.has(orcidKey)) { + byOrcid.set(orcidKey, existing); + } + continue; + } + } + + const likelyMatch = deduped.find((existing) => { + const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); + const hasConflictingOrcid = existingOrcid && orcidKey && existingOrcid !== orcidKey; + + if (hasConflictingOrcid) { + return false; + } + + return authorsLikelyMatch(existing, author); + }); + + if (likelyMatch) { + if (!likelyMatch.orcid && author.orcid) { + likelyMatch.orcid = author.orcid; + } + if (!likelyMatch.affiliation && author.affiliation) { + likelyMatch.affiliation = author.affiliation; + } + + const mergedOrcidKey = cleanString(likelyMatch?.orcid ?? '').toLowerCase(); + if (mergedOrcidKey && !byOrcid.has(mergedOrcidKey)) { + byOrcid.set(mergedOrcidKey, likelyMatch); + } + continue; + } + + if (orcidKey && byOrcid.has(orcidKey)) { + const existing = byOrcid.get(orcidKey); + if (!existing.affiliation && author.affiliation) { + existing.affiliation = author.affiliation; + } + continue; + } + + const key = [ + cleanString(author?.givenNames ?? '').toLowerCase(), + cleanString(author?.familyNames ?? '').toLowerCase(), + cleanString(author?.orcid ?? '').toLowerCase(), + ].join('|'); + + if (!key || seen.has(key)) { + continue; + } + + seen.add(key); + const normalizedAuthor = { ...author }; + deduped.push(normalizedAuthor); + + if (orcidKey) { + byOrcid.set(orcidKey, normalizedAuthor); + } + + if (nameKey !== '|') { + byName.set(nameKey, normalizedAuthor); + } + } + + return deduped; +} + +export function orderAuthorsByContributorRank(authors, contributorAuthors) { + const normalizedAuthors = normalizeAuthors(Array.isArray(authors) ? authors : []); + const normalizedContributors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); + + if (normalizedContributors.length === 0 || normalizedAuthors.length <= 1) { + return normalizedAuthors; + } + + const contributorOrcidIndex = new Map(); + const contributorNameKeyIndex = new Map(); + + normalizedContributors.forEach((contributorAuthor, index) => { + const orcidKey = cleanString(contributorAuthor?.orcid ?? '').toLowerCase(); + if (orcidKey && !contributorOrcidIndex.has(orcidKey)) { + contributorOrcidIndex.set(orcidKey, index); + } + + for (const key of authorAltNameKeys(contributorAuthor)) { + if (!contributorNameKeyIndex.has(key)) { + contributorNameKeyIndex.set(key, index); + } + } + }); + + const ranked = normalizedAuthors.map((author, originalIndex) => { + const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); + if (orcidKey && contributorOrcidIndex.has(orcidKey)) { + return { author, originalIndex, rank: contributorOrcidIndex.get(orcidKey) }; + } + + const nameRanks = authorAltNameKeys(author) + .map((key) => contributorNameKeyIndex.get(key)) + .filter((value) => Number.isInteger(value)); + + if (nameRanks.length > 0) { + return { author, originalIndex, rank: Math.min(...nameRanks) }; + } + + const heuristicIndex = normalizedContributors.findIndex((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); + if (heuristicIndex >= 0) { + return { author, originalIndex, rank: heuristicIndex }; + } + + return { author, originalIndex, rank: Number.POSITIVE_INFINITY }; + }); + + ranked.sort((left, right) => { + if (left.rank !== right.rank) { + return left.rank - right.rank; + } + + return left.originalIndex - right.originalIndex; + }); + + return ranked.map((entry) => entry.author); +} diff --git a/src/services/githubImporterContributors.js b/src/services/githubImporterContributors.js new file mode 100644 index 0000000..f9f91fd --- /dev/null +++ b/src/services/githubImporterContributors.js @@ -0,0 +1,258 @@ +const API_BASE = 'https://api.github.com'; +const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; +const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; +const GITHUB_PAGE_SIZE = 100; + +async function fetchOrcidFromGithubProfileHtml(profileUrl, cleanString, extractOrcidFromGithubHtml) { + const url = cleanString(profileUrl); + if (!url) { + return null; + } + + try { + const response = await fetch(url, { + headers: { + Accept: 'text/html', + }, + }); + + if (!response.ok) { + return null; + } + + const html = await response.text(); + return extractOrcidFromGithubHtml(html); + } catch { + return null; + } +} + +function isAutomatedContributor(contributor, profile, cleanString) { + const login = cleanString(profile?.login ?? contributor?.login ?? '').toLowerCase(); + const contributorType = cleanString(contributor?.type ?? '').toLowerCase(); + const profileType = cleanString(profile?.type ?? '').toLowerCase(); + + if ((contributorType && contributorType !== 'user') || (profileType && profileType !== 'user')) { + return true; + } + + if (!login) { + return false; + } + + if (login.endsWith('[bot]')) { + return true; + } + + return /(^|[-_])(github-actions|dependabot|copilot|codex|claude|swe-agent)([-_]|$)/.test(login); +} + +async function fetchAllContributors(owner, repo, warnings, authToken, maxContributors, { fetchOptionalJson, addWarning }) { + const contributors = []; + let page = 1; + + while (true) { + const pageContributors = await fetchOptionalJson( + `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, + { + authToken, + source: 'contributors', + label: `contributors page ${page}`, + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }, + ) || []; + + if (!Array.isArray(pageContributors) || pageContributors.length === 0) { + break; + } + + contributors.push(...pageContributors); + + if (maxContributors && contributors.length >= maxContributors) { + return contributors.slice(0, maxContributors); + } + + if (pageContributors.length < GITHUB_PAGE_SIZE) { + break; + } + + page += 1; + } + + return contributors; +} + +export function resolveContributorFallbackLimit(options = {}) { + if (!Object.prototype.hasOwnProperty.call(options, 'contributorFallbackLimit')) { + return TOP_CONTRIBUTOR_FALLBACK_LIMIT; + } + + if (options.contributorFallbackLimit == null || options.contributorFallbackLimit === '') { + return null; + } + + const rawLimit = Number(options.contributorFallbackLimit); + + if (!Number.isFinite(rawLimit)) { + return TOP_CONTRIBUTOR_FALLBACK_LIMIT; + } + + return Math.min(Math.max(Math.trunc(rawLimit), 1), MAX_CONTRIBUTOR_FALLBACK_LIMIT); +} + +export async function fetchContributorAuthors({ + owner, + repo, + warnings, + authToken = '', + contributorFallbackLimit = TOP_CONTRIBUTOR_FALLBACK_LIMIT, + emitFallbackWarning = true, + cleanString, + normalizeAuthor, + normalizeAuthors, + dedupeAuthors, + addWarning, + fetchOptionalJson, + extractOrcidFromGithubProfile, + extractOrcidFromGithubHtml, +}) { + const contributors = await fetchAllContributors(owner, repo, warnings, authToken, contributorFallbackLimit, { + fetchOptionalJson, + addWarning, + }); + + if (!Array.isArray(contributors) || contributors.length === 0) { + return { + fallbackAuthors: [], + lookupAuthors: [], + }; + } + + if (emitFallbackWarning) { + addWarning( + warnings, + 'authors', + 'commit-based-fallback', + contributorFallbackLimit + ? `Using top ${contributorFallbackLimit} contributors as fallback authors.` + : 'Using contributors as fallback authors.', + { owner, repo }, + ); + } + + const profiles = await Promise.all( + contributors.map(async (contributor) => { + const login = cleanString(contributor?.login ?? ''); + if (!login) { + return { + contributor, + profile: null, + socialAccounts: [], + author: null, + autoFilledOrcid: false, + excludedAutomated: false, + }; + } + + const profile = await fetchOptionalJson( + `${API_BASE}/users/${encodeURIComponent(login)}`, + { + authToken, + source: 'contributor-profile', + label: `the profile for ${login}`, + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }, + ); + + const socialAccounts = await fetchOptionalJson( + `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, + { + authToken, + source: 'contributor-profile-links', + label: `the profile links for ${login}`, + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }, + ) || []; + + if (isAutomatedContributor(contributor, profile, cleanString)) { + return { + contributor, + profile, + socialAccounts, + author: null, + autoFilledOrcid: false, + excludedAutomated: true, + }; + } + + let profileOrcid = extractOrcidFromGithubProfile(profile, socialAccounts); + if (!profileOrcid) { + profileOrcid = await fetchOrcidFromGithubProfileHtml( + profile?.html_url ?? contributor?.html_url ?? '', + cleanString, + extractOrcidFromGithubHtml, + ); + } + + if (profile?.name) { + return { + contributor, + profile, + socialAccounts, + author: normalizeAuthor({ + name: profile.name, + affiliation: profile.company ?? '', + orcid: profileOrcid, + }), + autoFilledOrcid: Boolean(profileOrcid), + excludedAutomated: false, + }; + } + + return { + contributor, + profile, + socialAccounts, + author: normalizeAuthor({ + name: login, + affiliation: '', + orcid: profileOrcid, + }), + autoFilledOrcid: Boolean(profileOrcid), + excludedAutomated: false, + }; + }), + ); + + const excludedAutomatedCount = profiles.filter((entry) => entry?.excludedAutomated).length; + if (excludedAutomatedCount > 0) { + addWarning( + warnings, + 'authors', + 'automated-contributors-excluded', + `Excluded ${excludedAutomatedCount} automated account(s) from fallback authors.`, + { owner, repo }, + ); + } + + const autoFilledOrcidCount = profiles.filter((entry) => entry?.autoFilledOrcid).length; + if (autoFilledOrcidCount > 0) { + addWarning( + warnings, + 'authors', + 'orcid-autofilled', + `Auto-filled ORCID for ${autoFilledOrcidCount} contributor(s) from GitHub profile data.`, + { owner, repo }, + ); + } + + const fallbackAuthors = profiles + .slice(0, contributorFallbackLimit ?? profiles.length) + .map((entry) => entry?.author); + const lookupAuthors = profiles.map((entry) => entry?.author); + + return { + fallbackAuthors: dedupeAuthors(normalizeAuthors(fallbackAuthors)), + lookupAuthors: dedupeAuthors(normalizeAuthors(lookupAuthors)), + }; +} diff --git a/src/services/githubImporterParsers.js b/src/services/githubImporterParsers.js new file mode 100644 index 0000000..a985795 --- /dev/null +++ b/src/services/githubImporterParsers.js @@ -0,0 +1,170 @@ +import { + cleanString as utilCleanString, + extractFirstMarkdownParagraph as utilExtractFirstMarkdownParagraph, + normalizeAuthor as utilNormalizeAuthor, + normalizeKeywords as utilNormalizeKeywords, + normalizeRepoUrl as utilNormalizeRepoUrl, +} from './githubImporterUtils.js'; + +const cleanString = utilCleanString; +const normalizeAuthor = utilNormalizeAuthor; +const normalizeKeywords = utilNormalizeKeywords; +const normalizeRepoUrl = utilNormalizeRepoUrl; +const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; + +function parseJsonSafely(text) { + return JSON.parse(text); +} + +function extractPackageAuthors(payload) { + const candidates = []; + + if (payload.author) { + candidates.push(payload.author); + } + + if (Array.isArray(payload.authors)) { + candidates.push(...payload.authors); + } + + return candidates.map((item) => normalizeAuthor(item)).filter(Boolean); +} + +export function parsePackageJson(text) { + const payload = parseJsonSafely(text); + + return { + title: cleanString(payload.name ?? ''), + abstract: cleanString(payload.description ?? ''), + version: cleanString(payload.version ?? ''), + repositoryCode: normalizeRepoUrl(typeof payload.repository === 'string' ? payload.repository : payload.repository?.url ?? payload.homepage ?? ''), + license: cleanString(typeof payload.license === 'string' ? payload.license : payload.license?.type ?? ''), + keywords: normalizeKeywords(payload.keywords), + authors: extractPackageAuthors(payload), + }; +} + +function extractTomlSection(text, sectionName) { + const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); + const sectionLines = []; + let inSection = false; + + for (const line of lines) { + const sectionMatch = line.trim().match(/^\[([^\]]+)\]$/); + + if (sectionMatch) { + if (inSection) { + break; + } + + inSection = sectionMatch[1] === sectionName; + continue; + } + + if (inSection) { + sectionLines.push(line); + } + } + + return sectionLines.join('\n'); +} + +function extractTomlValue(sectionText, key) { + const match = sectionText.match(new RegExp(`^${key}\\s*=\\s*(.+)$`, 'm')); + return match ? match[1].trim() : ''; +} + +function parseTomlString(sectionText, key) { + const value = extractTomlValue(sectionText, key); + const match = value.match(/^['"](.+?)['"]$/); + return match ? match[1].trim() : ''; +} + +function parseTomlStrings(value) { + return [...String(value ?? '').matchAll(/['"]([^'"]+)['"]/g)].map((match) => cleanString(match[1])).filter(Boolean); +} + +export function parsePyprojectToml(text) { + const section = extractTomlSection(text, 'project') || extractTomlSection(text, 'tool.poetry'); + const authorsBlock = extractTomlValue(section, 'authors') || extractTomlValue(section, 'maintainers'); + const authors = [...String(authorsBlock ?? '').matchAll(/name\s*=\s*['"]([^'"]+)['"]/g)] + .map((match) => normalizeAuthor({ name: match[1] })) + .filter(Boolean); + + const licenseValue = parseTomlString(section, 'license') || cleanString((section.match(/license\s*=\s*\{[^}]*text\s*=\s*['"]([^'"]+)['"][^}]*\}/s) || [])[1] ?? ''); + const repositoryCode = parseTomlString(section, 'repository') || parseTomlString(section, 'homepage') || parseTomlString(section, 'url'); + + return { + title: parseTomlString(section, 'name'), + abstract: parseTomlString(section, 'description'), + version: parseTomlString(section, 'version'), + repositoryCode, + license: licenseValue, + keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), + authors, + }; +} + +export function parseSetupPy(text) { + const source = String(text ?? ''); + const extract = (key) => cleanString((source.match(new RegExp(`${key}\\s*=\\s*['"]([^'"]+)['"]`, 'm')) || [])[1] ?? ''); + + const authors = []; + const author = extract('author'); + const maintainer = extract('maintainer'); + + if (author) { + authors.push(normalizeAuthor({ name: author })); + } else if (maintainer) { + authors.push(normalizeAuthor({ name: maintainer })); + } + + return { + title: extract('name'), + abstract: extract('description'), + version: extract('version'), + repositoryCode: normalizeRepoUrl(extract('url')), + license: extract('license'), + keywords: normalizeKeywords(extract('keywords')), + authors: authors.filter(Boolean), + }; +} + +export function parseCargoToml(text) { + const section = extractTomlSection(text, 'package'); + const authors = parseTomlStrings(extractTomlValue(section, 'authors')).map((name) => normalizeAuthor({ name })).filter(Boolean); + + return { + title: parseTomlString(section, 'name'), + abstract: parseTomlString(section, 'description'), + version: parseTomlString(section, 'version'), + repositoryCode: parseTomlString(section, 'repository'), + license: parseTomlString(section, 'license'), + keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), + authors, + }; +} + +export function parsePomXml(text) { + const source = String(text ?? ''); + const extract = (pattern) => cleanString((source.match(pattern) || [])[1] ?? ''); + const authors = [...source.matchAll(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/developer>/g)] + .map((match) => normalizeAuthor({ name: match[1] })) + .filter(Boolean); + + const licenseMatch = source.match(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/license>/); + + return { + title: extract(/([^<]+)<\/name>/), + abstract: extract(/([^<]+)<\/description>/), + version: extract(/([^<]+)<\/version>/), + repositoryCode: extract(/([^<]+)<\/url>/), + license: cleanString((licenseMatch || [])[1] ?? ''), + keywords: [], + authors, + }; +} + +export function parseReadme(text) { + return extractFirstMarkdownParagraph(text); +} diff --git a/src/services/githubImporterUtils.js b/src/services/githubImporterUtils.js new file mode 100644 index 0000000..e45ac27 --- /dev/null +++ b/src/services/githubImporterUtils.js @@ -0,0 +1,271 @@ +import { normalizeOrcid } from '../utils/orcid.js'; + +function cleanString(value) { + return String(value ?? '').replace(/[\t ]+/g, ' ').trim(); +} + +function firstNonEmpty(...values) { + for (const value of values) { + if (Array.isArray(value)) { + if (value.length > 0) { + return value; + } + continue; + } + + const text = cleanString(value); + if (text) { + return text; + } + } + + return ''; +} + +function normalizeStringList(value) { + if (Array.isArray(value)) { + return value.map((item) => cleanString(item)).filter(Boolean); + } + + if (!value) { + return []; + } + + return String(value) + .split(/[\n,]/) + .map((item) => cleanString(item)) + .filter(Boolean); +} + +function normalizeKeywords(value) { + return [...new Set(normalizeStringList(value).map((keyword) => keyword.toLowerCase()))]; +} + +function normalizeReferences(value) { + if (Array.isArray(value)) { + return value.map((item) => cleanString(item)).filter(Boolean); + } + + if (!value) { + return []; + } + + return String(value) + .split(/\n+/) + .map((item) => cleanString(item)) + .filter(Boolean); +} + +function normalizeGrants(value) { + if (Array.isArray(value)) { + return value + .map((item) => { + if (typeof item === 'string') { + return cleanString(item); + } + + if (item && typeof item === 'object') { + return cleanString(item.id ?? item.value ?? item.grantId ?? ''); + } + + return ''; + }) + .filter(Boolean); + } + + if (!value) { + return []; + } + + return String(value) + .split(/\n+/) + .map((item) => cleanString(item)) + .filter(Boolean); +} + +function capitalizeToken(token) { + const text = cleanString(token); + if (!text) { + return ''; + } + + return text + .split(/([\-'])/) + .map((part) => { + if (part === '-' || part === "'") { + return part; + } + + // Preserve mixed-case tokens (for example, McDonald) and normalize others. + if (/[a-z]/.test(part) && /[A-Z]/.test(part)) { + return part; + } + + const lower = part.toLowerCase(); + return lower.charAt(0).toUpperCase() + lower.slice(1); + }) + .join(''); +} + +function capitalizeName(value) { + return cleanString(value) + .split(/\s+/) + .map((part) => capitalizeToken(part)) + .filter(Boolean) + .join(' '); +} + +function humanizeIdentifier(value) { + return cleanString(value) + .replace(/[._-]+/g, ' ') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .replace(/([a-z\d])([A-Z])/g, '$1 $2') + .replace(/([A-Za-z])(\d)/g, '$1 $2') + .replace(/(\d)([A-Za-z])/g, '$1 $2'); +} + +function splitDisplayName(name) { + const value = cleanString(name); + + if (!value) { + return { givenNames: '', familyNames: '' }; + } + + if (value.includes(',')) { + const [familyNames, ...givenParts] = value.split(','); + return { + givenNames: capitalizeName(givenParts.join(',').trim()), + familyNames: capitalizeName(familyNames), + }; + } + + const normalized = humanizeIdentifier(value); + const parts = normalized.split(/\s+/).filter(Boolean); + + if (parts.length <= 1) { + return { givenNames: capitalizeName(parts[0] ?? ''), familyNames: '' }; + } + + return { + givenNames: capitalizeName(parts.slice(0, -1).join(' ')), + familyNames: capitalizeName(parts[parts.length - 1]), + }; +} + +function normalizeAuthor(input) { + if (!input) { + return null; + } + + if (typeof input === 'string') { + const { givenNames, familyNames } = splitDisplayName(input); + return givenNames || familyNames ? { givenNames, familyNames, orcid: '', affiliation: '' } : null; + } + + if (typeof input !== 'object') { + return null; + } + + const name = cleanString(input.name ?? input.fullName ?? input.full_name ?? input.creator_name ?? ''); + const parsedName = name ? splitDisplayName(name) : null; + let givenNames = capitalizeName(input.givenNames ?? input['given-names'] ?? input.firstName ?? input.firstname ?? parsedName?.givenNames ?? ''); + let familyNames = capitalizeName(input.familyNames ?? input['family-names'] ?? input.lastName ?? input.lastname ?? parsedName?.familyNames ?? ''); + const affiliation = cleanString(input.affiliation ?? input.organization ?? input.company ?? input.institution ?? ''); + const orcid = normalizeOrcid(input.orcid ?? input.ORCID ?? input.orcidId ?? ''); + + if (givenNames && !familyNames) { + const reparsed = splitDisplayName(givenNames); + if (reparsed.familyNames) { + givenNames = reparsed.givenNames; + familyNames = reparsed.familyNames; + } + } + + if (!givenNames && !familyNames && !affiliation && !orcid) { + return null; + } + + return { givenNames, familyNames, orcid, affiliation }; +} + +function normalizeAuthors(value) { + if (!Array.isArray(value)) { + return []; + } + + return value.map((item) => normalizeAuthor(item)).filter(Boolean); +} + +function normalizeRepoUrl(value) { + const text = cleanString(value); + if (!text) { + return ''; + } + + const trimmed = text.replace(/^git\+/, '').replace(/\.git$/i, '').replace(/\/+$/, ''); + + try { + const parsed = new URL(trimmed); + const host = parsed.hostname.toLowerCase(); + let pathname = parsed.pathname.replace(/\/+$/, ''); + if (host === 'github.com') { + pathname = pathname.toLowerCase(); + } + return `${parsed.protocol}//${host}${pathname}`; + } catch { + return trimmed; + } +} + +function normalizeVersionForCompare(value) { + const text = cleanString(value).toLowerCase(); + if (!text) { + return ''; + } + + return text.replace(/^v(?=\d)/, ''); +} + +function extractFirstMarkdownParagraph(text) { + const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); + const paragraph = []; + let started = false; + + for (const line of lines) { + const trimmed = line.trim(); + + if (!trimmed) { + if (started) { + break; + } + continue; + } + + if (!started && /^#{1,6}\s+/.test(trimmed)) { + started = true; + continue; + } + + if (!started && /^(!|\[|-)/.test(trimmed)) { + continue; + } + + started = true; + paragraph.push(trimmed); + } + + return paragraph.join(' ').replace(/\s+/g, ' ').trim(); +} + +export { + cleanString, + extractFirstMarkdownParagraph, + firstNonEmpty, + normalizeAuthor, + normalizeAuthors, + normalizeGrants, + normalizeKeywords, + normalizeReferences, + normalizeRepoUrl, + normalizeVersionForCompare, +}; diff --git a/tests/services/citationValidation.test.js b/tests/services/citationValidation.test.js index b461cb5..a21316c 100644 --- a/tests/services/citationValidation.test.js +++ b/tests/services/citationValidation.test.js @@ -67,7 +67,15 @@ repository-code: "https://github.com/Imageomics/OpenCite" test('toCitationCff omits empty author name fields in references', () => { const output = toCitationCff({ title: 'OpenCite', - authors: [], + authors: [ + { + citationAuthor: { + 'given-names': 'Jane', + 'family-names': 'Doe', + orcid: '', + }, + }, + ], keywords: [], license: 'MIT', typeOfWork: 'software', From cd9040978495787120eedbd6378c7eb16525a31a Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Wed, 12 Aug 2026 11:32:59 -0400 Subject: [PATCH 02/23] refactor: extract repeated importer helpers --- src/services/githubImporter.js | 43 ++++++++++++--------------- src/services/githubImporterUtils.js | 10 +++++++ tests/services/githubImporter.test.js | 8 +++++ 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index e2b85f3..23f17d1 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -21,6 +21,7 @@ import { normalizeReferences as utilNormalizeReferences, normalizeRepoUrl as utilNormalizeRepoUrl, normalizeVersionForCompare as utilNormalizeVersionForCompare, + stripWrappingQuotes as utilStripWrappingQuotes, } from './githubImporterUtils.js'; import { fetchContributorAuthors, @@ -64,6 +65,7 @@ const normalizeAuthor = utilNormalizeAuthor; const normalizeAuthors = utilNormalizeAuthors; const normalizeRepoUrl = utilNormalizeRepoUrl; const normalizeVersionForCompare = utilNormalizeVersionForCompare; +const stripWrappingQuotes = utilStripWrappingQuotes; const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; function makeIssue(kind, source, code, message, details = {}) { @@ -100,6 +102,19 @@ function shouldInspectRepositoryFiles(options = {}) { return options.inspectRepositoryFiles !== false; } +function addValidationWarnings(warnings, metaKind, path, validationResult) { + if (!validationResult || validationResult.isValid) { + return; + } + + const message = `${path} failed validation: ${validationResult.errors.join(' | ')}`; + addWarning(warnings, metaKind, `${metaKind}-file-invalid`, message, { path }); + + for (const warning of validationResult.warnings) { + addWarning(warnings, metaKind, `${metaKind}-file-warning`, `${path}: ${warning}`, { path }); + } +} + export function resolvePreferredCitationPath(fileContents = {}) { if (fileContents['CITATION.cff']) { return 'CITATION.cff'; @@ -140,18 +155,8 @@ export function summarizeImportedMetadataFiles(fileContents = {}) { if (!citationValidation.isValid) { summary.citation.valid = false; summary.citation.errors = [...citationValidation.errors]; - addWarning( - warnings, - 'citation', - 'citation-file-invalid', - `${preferredCitationPath} failed validation: ${citationValidation.errors.join(' | ')}`, - { path: preferredCitationPath }, - ); - } - - for (const warning of citationValidation.warnings) { - addWarning(warnings, 'citation', 'citation-file-warning', `${preferredCitationPath}: ${warning}`, { path: preferredCitationPath }); } + addValidationWarnings(warnings, 'citation', preferredCitationPath, citationValidation); } const zenodoPath = '.zenodo.json'; @@ -162,18 +167,8 @@ export function summarizeImportedMetadataFiles(fileContents = {}) { if (!zenodoValidation.isValid) { summary.zenodo.valid = false; summary.zenodo.errors = [...zenodoValidation.errors]; - addWarning( - warnings, - 'zenodo', - 'zenodo-file-invalid', - `${zenodoPath} failed validation: ${zenodoValidation.errors.join(' | ')}`, - { path: zenodoPath }, - ); - } - - for (const warning of zenodoValidation.warnings) { - addWarning(warnings, 'zenodo', 'zenodo-file-warning', `${zenodoPath}: ${warning}`, { path: zenodoPath }); } + addValidationWarnings(warnings, 'zenodo', zenodoPath, zenodoValidation); } return summary; @@ -246,7 +241,7 @@ export function parseCitationCff(text) { }; const assignScalar = (key, value) => { - const normalized = cleanString(value).replace(/^"|"$/g, ''); + const normalized = stripWrappingQuotes(value); if (!normalized) { return; @@ -327,7 +322,7 @@ export function parseCitationCff(text) { if (trimmed.startsWith('-')) { flushReference(); - const inline = cleanString(trimmed.slice(1)).replace(/^"|"$/g, ''); + const inline = stripWrappingQuotes(trimmed.slice(1)); if (!inline) { currentReference = {}; continue; diff --git a/src/services/githubImporterUtils.js b/src/services/githubImporterUtils.js index e45ac27..42a0c6e 100644 --- a/src/services/githubImporterUtils.js +++ b/src/services/githubImporterUtils.js @@ -4,6 +4,15 @@ function cleanString(value) { return String(value ?? '').replace(/[\t ]+/g, ' ').trim(); } +function stripWrappingQuotes(value) { + const text = cleanString(value); + if (!text) { + return ''; + } + + return text.replace(/^"|"$/g, '').replace(/^'|'$/g, ''); +} + function firstNonEmpty(...values) { for (const value of values) { if (Array.isArray(value)) { @@ -268,4 +277,5 @@ export { normalizeReferences, normalizeRepoUrl, normalizeVersionForCompare, + stripWrappingQuotes, }; diff --git a/tests/services/githubImporter.test.js b/tests/services/githubImporter.test.js index cac8f04..5984788 100644 --- a/tests/services/githubImporter.test.js +++ b/tests/services/githubImporter.test.js @@ -10,6 +10,7 @@ import { summarizeImportedMetadataFiles, validateImportedMetadataFiles, } from '../../src/services/githubImporter.js'; +import { stripWrappingQuotes } from '../../src/services/githubImporterUtils.js'; test('parseCitationCff extracts top-level fields from common CFF content', () => { const parsed = parseCitationCff(`cff-version: 1.2.0 @@ -35,6 +36,13 @@ authors: assert.equal(parsed.authors.length, 1); }); +test('stripWrappingQuotes removes matching quote wrappers without altering inner text', () => { + assert.equal(stripWrappingQuotes('"OpenCite"'), 'OpenCite'); + assert.equal(stripWrappingQuotes("'OpenCite'"), 'OpenCite'); + assert.equal(stripWrappingQuotes('OpenCite'), 'OpenCite'); + assert.equal(stripWrappingQuotes('"quoted \\"text\\""'), 'quoted \\"text\\"'); +}); + test('parseCitationCff emits warning for preferred-citation sections', () => { const parsed = parseCitationCff(`cff-version: 1.2.0 title: "OpenCite" From d4c7625848c65ad11b5ba6d1184428bda336c366 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:11:08 -0400 Subject: [PATCH 03/23] refactor: integrate buildGithubRequestConfig into GitHub metadata importer and contributors --- src/services/githubApi.js | 8 ++++++-- src/services/githubImporter.js | 23 ++++++++++++++-------- src/services/githubImporterContributors.js | 14 +++++++------ tests/services/githubImporter.test.js | 18 +++++++++++++++++ 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/services/githubApi.js b/src/services/githubApi.js index 44d5fc9..35fbbf2 100644 --- a/src/services/githubApi.js +++ b/src/services/githubApi.js @@ -92,6 +92,10 @@ export function resolveGithubToken(options = {}) { return ''; } +export function buildGithubRequestConfig({ authToken = '', source = '', label = '', onWarning = () => {} } = {}) { + return { authToken, source, label, onWarning }; +} + export function createGithubHeaders(token = '') { const headers = { Accept: 'application/vnd.github+json', @@ -179,12 +183,12 @@ export async function fetchLatestCommitDate(owner, repo, defaultBranch, { authTo const branchFilter = defaultBranch ? `&sha=${encodeURIComponent(defaultBranch)}` : ''; const commits = await fetchOptionalJson( `${API_BASE}/repos/${owner}/${repo}/commits?per_page=1${branchFilter}`, - { + buildGithubRequestConfig({ authToken, source: 'commits', label: 'the latest commit', onWarning, - }, + }), ); if (!Array.isArray(commits) || commits.length === 0) { diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 23f17d1..d7596b5 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -3,6 +3,7 @@ import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile } from '../ut import { validateCitationCffText } from './citationValidation.js'; import { runCitationHealthScan } from './citationHealthScan.js'; import { + buildGithubRequestConfig, fetchContentsFile, fetchLatestCommitDate, fetchOptionalJson, @@ -704,12 +705,15 @@ export async function importGithubMetadata(repoUrl, options = {}) { } const defaultBranch = cleanString(repoData.default_branch ?? ''); - const releaseData = await fetchOptionalJson(`${API_BASE}/repos/${owner}/${repo}/releases/latest`, { - authToken, - source: 'release', - label: 'the latest release', - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }); + const releaseData = await fetchOptionalJson( + `${API_BASE}/repos/${owner}/${repo}/releases/latest`, + buildGithubRequestConfig({ + authToken, + source: 'release', + label: 'the latest release', + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }), + ); const latestCommitDate = releaseData?.published_at ? '' : await fetchLatestCommitDate(owner, repo, defaultBranch, { @@ -724,12 +728,15 @@ export async function importGithubMetadata(repoUrl, options = {}) { if (inspectRepositoryFiles) { const branchInfo = defaultBranch - ? await fetchOptionalJson(`${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(defaultBranch)}`, { + ? await fetchOptionalJson( + `${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(defaultBranch)}`, + buildGithubRequestConfig({ authToken, source: 'branch', label: 'the default branch', onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }) + }), + ) : null; ref = cleanString(branchInfo?.name ?? defaultBranch ?? repoData.default_branch ?? 'HEAD'); diff --git a/src/services/githubImporterContributors.js b/src/services/githubImporterContributors.js index f9f91fd..83a7d95 100644 --- a/src/services/githubImporterContributors.js +++ b/src/services/githubImporterContributors.js @@ -1,3 +1,5 @@ +import { buildGithubRequestConfig } from './githubApi.js'; + const API_BASE = 'https://api.github.com'; const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; @@ -54,12 +56,12 @@ async function fetchAllContributors(owner, repo, warnings, authToken, maxContrib while (true) { const pageContributors = await fetchOptionalJson( `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, - { + buildGithubRequestConfig({ authToken, source: 'contributors', label: `contributors page ${page}`, onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, + }), ) || []; if (!Array.isArray(pageContributors) || pageContributors.length === 0) { @@ -156,22 +158,22 @@ export async function fetchContributorAuthors({ const profile = await fetchOptionalJson( `${API_BASE}/users/${encodeURIComponent(login)}`, - { + buildGithubRequestConfig({ authToken, source: 'contributor-profile', label: `the profile for ${login}`, onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, + }), ); const socialAccounts = await fetchOptionalJson( `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, - { + buildGithubRequestConfig({ authToken, source: 'contributor-profile-links', label: `the profile links for ${login}`, onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, + }), ) || []; if (isAutomatedContributor(contributor, profile, cleanString)) { diff --git a/tests/services/githubImporter.test.js b/tests/services/githubImporter.test.js index 5984788..1a63d0f 100644 --- a/tests/services/githubImporter.test.js +++ b/tests/services/githubImporter.test.js @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { buildGithubRequestConfig } from '../../src/services/githubApi.js'; import { addCitationConsistencyWarnings, importGithubMetadata, @@ -36,6 +37,23 @@ authors: assert.equal(parsed.authors.length, 1); }); +test('buildGithubRequestConfig returns the same GitHub request-field shape', () => { + const onWarning = () => {}; + const config = buildGithubRequestConfig({ + authToken: 'token-123', + source: 'release', + label: 'the latest release', + onWarning, + }); + + assert.deepEqual(config, { + authToken: 'token-123', + source: 'release', + label: 'the latest release', + onWarning, + }); +}); + test('stripWrappingQuotes removes matching quote wrappers without altering inner text', () => { assert.equal(stripWrappingQuotes('"OpenCite"'), 'OpenCite'); assert.equal(stripWrappingQuotes("'OpenCite'"), 'OpenCite'); From 503c0775a05ef985a30f83ac4bb7780db5c96e94 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:25:35 -0400 Subject: [PATCH 04/23] refactor: modularize GitHub API calls and improve metadata handling in importer --- src/services/github.js | 11 +- src/services/githubApi.js | 33 ++++ src/services/githubImporter.js | 187 +-------------------- src/services/githubImporterContributors.js | 16 +- src/services/githubImporterMerge.js | 159 ++++++++++++++++++ 5 files changed, 217 insertions(+), 189 deletions(-) create mode 100644 src/services/githubImporterMerge.js diff --git a/src/services/github.js b/src/services/github.js index c6053ac..f509ce5 100644 --- a/src/services/github.js +++ b/src/services/github.js @@ -1,4 +1,9 @@ import { createMetadata } from '../core/metadataModel.js'; +import { + buildGithubCommitListApiUrl, + buildGithubReleaseApiUrl, + buildGithubRepoApiUrl, +} from './githubApi.js'; /** * Parse a GitHub repository URL to extract owner and repo name @@ -37,7 +42,7 @@ function parseGithubUrl(url) { * @throws {Error} if API request fails */ async function fetchRepoData(owner, repo) { - const url = `https://api.github.com/repos/${owner}/${repo}`; + const url = buildGithubRepoApiUrl(owner, repo); const response = await fetch(url, { headers: { Accept: 'application/vnd.github.v3+json', @@ -61,7 +66,7 @@ async function fetchRepoData(owner, repo) { * @returns {Promise} Latest release object or null if no releases */ async function fetchLatestRelease(owner, repo) { - const url = `https://api.github.com/repos/${owner}/${repo}/releases/latest`; + const url = buildGithubReleaseApiUrl(owner, repo); const response = await fetch(url, { headers: { Accept: 'application/vnd.github.v3+json', @@ -95,7 +100,7 @@ async function fetchDefaultBranchSha(owner, repo, defaultBranch) { return null; } - const url = `https://api.github.com/repos/${owner}/${repo}/commits?sha=${encodeURIComponent(defaultBranch)}&per_page=1`; + const url = buildGithubCommitListApiUrl(owner, repo, defaultBranch); const response = await fetch(url, { headers: { Accept: 'application/vnd.github.v3+json', diff --git a/src/services/githubApi.js b/src/services/githubApi.js index 35fbbf2..3274196 100644 --- a/src/services/githubApi.js +++ b/src/services/githubApi.js @@ -96,6 +96,39 @@ export function buildGithubRequestConfig({ authToken = '', source = '', label = return { authToken, source, label, onWarning }; } +export function buildGithubRepoApiUrl(owner, repo) { + return `${API_BASE}/repos/${owner}/${repo}`; +} + +export function buildGithubReleaseApiUrl(owner, repo) { + return `${API_BASE}/repos/${owner}/${repo}/releases/latest`; +} + +export function buildGithubCommitListApiUrl(owner, repo, defaultBranch = '') { + const branchFilter = defaultBranch ? `&sha=${encodeURIComponent(defaultBranch)}` : ''; + return `${API_BASE}/repos/${owner}/${repo}/commits?per_page=1${branchFilter}`; +} + +export function buildGithubBranchApiUrl(owner, repo, branch) { + return `${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`; +} + +export function buildGithubContentsApiUrl(owner, repo, path, ref) { + return `${API_BASE}/repos/${owner}/${repo}/contents/${encodePath(path)}?ref=${encodeURIComponent(ref)}`; +} + +export function buildGithubContributorsApiUrl(owner, repo, page, perPage = 100) { + return `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${perPage}&page=${page}`; +} + +export function buildGithubUserApiUrl(login) { + return `${API_BASE}/users/${encodeURIComponent(login)}`; +} + +export function buildGithubUserSocialAccountsApiUrl(login) { + return `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`; +} + export function createGithubHeaders(token = '') { const headers = { Accept: 'application/vnd.github+json', diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index d7596b5..0394c0d 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -3,6 +3,8 @@ import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile } from '../ut import { validateCitationCffText } from './citationValidation.js'; import { runCitationHealthScan } from './citationHealthScan.js'; import { + buildGithubBranchApiUrl, + buildGithubReleaseApiUrl, buildGithubRequestConfig, fetchContentsFile, fetchLatestCommitDate, @@ -28,11 +30,8 @@ import { fetchContributorAuthors, resolveContributorFallbackLimit, } from './githubImporterContributors.js'; -import { - dedupeAuthors, - enrichAuthorsWithContributorData, - orderAuthorsByContributorRank, -} from './githubImporterAuthors.js'; +import { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; +export { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; import { parseCargoToml, parsePackageJson, @@ -45,7 +44,6 @@ import { compareExistingMetadataFiles } from './metadataComparison.js'; import { runMetadataReviewPipeline } from './metadataReview.js'; import { validateZenodoJsonText } from './zenodoValidation.js'; -const API_BASE = 'https://api.github.com'; const FILES_TO_INSPECT = [ 'CITATION.cff', '.zenodo.json', @@ -61,11 +59,9 @@ const cleanString = utilCleanString; const firstNonEmpty = utilFirstNonEmpty; const normalizeKeywords = utilNormalizeKeywords; const normalizeReferences = utilNormalizeReferences; -const normalizeGrants = utilNormalizeGrants; const normalizeAuthor = utilNormalizeAuthor; const normalizeAuthors = utilNormalizeAuthors; const normalizeRepoUrl = utilNormalizeRepoUrl; -const normalizeVersionForCompare = utilNormalizeVersionForCompare; const stripWrappingQuotes = utilStripWrappingQuotes; const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; @@ -509,174 +505,6 @@ function parseFile(path, text, warnings, errors) { return null; } -function mapTypeOfWork(value) { - const type = cleanString(value).toLowerCase(); - - if (type === 'dataset') { - return 'dataset'; - } - - if (['article', 'book', 'book-chapter', 'conference-paper', 'journal-article', 'manuscript', 'preprint', 'report', 'thesis'].includes(type)) { - return 'article'; - } - - if (type === 'other') { - return 'other'; - } - - return 'software'; -} - -function mergeMetadata({ - repo, - release, - defaultPublicationDate, - citation, - zenodo, - packageMeta, - readme, - contributors, - contributorLookupAuthors, - supplementalCitationAuthors = [], -}) { - const primaryAuthors = [ - ...normalizeAuthors(Array.isArray(citation?.authors) ? citation.authors : []), - ...normalizeAuthors(Array.isArray(zenodo?.authors) ? zenodo.authors : []), - ...normalizeAuthors(Array.isArray(packageMeta?.authors) ? packageMeta.authors : []), - ...normalizeAuthors(Array.isArray(supplementalCitationAuthors) ? supplementalCitationAuthors : []), - ]; - const authors = [ - ...normalizeAuthors(primaryAuthors), - ...normalizeAuthors(Array.isArray(contributors) ? contributors : []), - ]; - const keywords = normalizeKeywords(firstNonEmpty(citation?.keywords, zenodo?.keywords, packageMeta?.keywords, repo?.topics)); - const references = normalizeReferences(firstNonEmpty(zenodo?.references, citation?.references)); - const grants = normalizeGrants(firstNonEmpty(zenodo?.grants)); - const enrichedAuthors = enrichAuthorsWithContributorData(authors, contributorLookupAuthors); - const dedupedAuthors = dedupeAuthors(enrichedAuthors); - const orderedAuthors = orderAuthorsByContributorRank(dedupedAuthors, contributorLookupAuthors); - - return createMetadata({ - title: cleanString(firstNonEmpty(citation?.title, zenodo?.title, packageMeta?.title, repo?.name)), - authors: orderedAuthors, - keywords, - license: cleanString(firstNonEmpty(citation?.license, zenodo?.license, packageMeta?.license, repo?.license?.spdx_id)), - typeOfWork: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), - customTypeOfWork: '', - zenodoUploadType: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), - // Prefer metadata file versions for pre-release authoring; fall back to latest release tag. - version: cleanString(firstNonEmpty(citation?.version, zenodo?.version, packageMeta?.version, release?.tag_name)), - publicationDate: cleanString(firstNonEmpty(release?.published_at, citation?.publicationDate, zenodo?.publicationDate, defaultPublicationDate)).split('T')[0], - repositoryCode: normalizeRepoUrl(firstNonEmpty(repo?.html_url, citation?.repositoryCode, packageMeta?.repositoryCode)), - doi: cleanString(firstNonEmpty(zenodo?.doi, citation?.doi)), - abstract: cleanString(firstNonEmpty(citation?.abstract, zenodo?.abstract, packageMeta?.abstract, readme, repo?.description)), - references, - grants, - }); -} - -export function addCitationConsistencyWarnings({ warnings, citation, zenodo, releaseData, repoData, metadata }) { - const releaseTag = cleanString(releaseData?.tag_name ?? ''); - const citationVersion = cleanString(citation?.version ?? ''); - const zenodoVersion = cleanString(zenodo?.version ?? ''); - const finalVersion = cleanString(metadata?.version ?? ''); - const normalizedReleaseTag = normalizeVersionForCompare(releaseTag); - const normalizedCitationVersion = normalizeVersionForCompare(citationVersion); - const normalizedZenodoVersion = normalizeVersionForCompare(zenodoVersion); - - if (normalizedReleaseTag && normalizedCitationVersion && normalizedReleaseTag !== normalizedCitationVersion) { - addWarning( - warnings, - 'citation', - 'version-mismatch', - `CITATION.cff version (${citationVersion}) differs from latest release tag (${releaseTag}); using CITATION.cff version for import.`, - ); - } - - if (normalizedReleaseTag && normalizedZenodoVersion && normalizedReleaseTag !== normalizedZenodoVersion) { - addWarning( - warnings, - 'zenodo', - 'version-mismatch', - `.zenodo.json version (${zenodoVersion}) differs from latest release tag (${releaseTag}); using .zenodo.json version for import.`, - ); - } - - if (normalizedCitationVersion && normalizedZenodoVersion && normalizedCitationVersion !== normalizedZenodoVersion) { - addWarning( - warnings, - 'citation', - 'cross-file-version-mismatch', - `CITATION.cff version (${citationVersion}) and .zenodo.json version (${zenodoVersion}) differ.`, - ); - } - - const citationDate = cleanString(citation?.publicationDate ?? '').split('T')[0]; - const zenodoDate = cleanString(zenodo?.publicationDate ?? '').split('T')[0]; - const releaseDate = cleanString(releaseData?.published_at ?? '').split('T')[0]; - - if (releaseDate && citationDate && releaseDate !== citationDate) { - addWarning( - warnings, - 'citation', - 'date-mismatch', - `CITATION.cff date-released (${citationDate}) differs from latest release date (${releaseDate}); using release date for import.`, - ); - } - - if (releaseDate && zenodoDate && releaseDate !== zenodoDate) { - addWarning( - warnings, - 'zenodo', - 'date-mismatch', - `.zenodo.json publication_date (${zenodoDate}) differs from latest release date (${releaseDate}); using release date for import.`, - ); - } - - const repoUrl = normalizeRepoUrl(repoData?.html_url ?? ''); - const citationRepoUrl = normalizeRepoUrl(citation?.repositoryCode ?? ''); - - if (repoUrl && citationRepoUrl && repoUrl !== citationRepoUrl) { - addWarning( - warnings, - 'citation', - 'repository-url-mismatch', - `CITATION.cff repository-code (${citationRepoUrl}) differs from repository URL (${repoUrl}); using repository URL for import.`, - ); - } - - if (!finalVersion) { - addWarning( - warnings, - 'citation', - 'missing-version', - 'No version could be determined from release tag, CITATION.cff, .zenodo.json, or package metadata.', - ); - } - - const repoSpdx = cleanString(repoData?.license?.spdx_id ?? '').toUpperCase(); - const citationLicense = cleanString(citation?.license ?? '').toUpperCase(); - const zenodoLicense = cleanString(zenodo?.license ?? '').toUpperCase(); - - if (repoSpdx && citationLicense && repoSpdx !== citationLicense) { - addWarning( - warnings, - 'citation', - 'license-mismatch', - `CITATION.cff license (${citationLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`, - ); - } - - if (repoSpdx && zenodoLicense && repoSpdx !== zenodoLicense) { - addWarning( - warnings, - 'zenodo', - 'license-mismatch', - `.zenodo.json license (${zenodoLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`, - ); - } -} - export async function importGithubMetadata(repoUrl, options = {}) { const warnings = []; const errors = []; @@ -695,7 +523,7 @@ export async function importGithubMetadata(repoUrl, options = {}) { return { metadata: emptyMetadata, warnings, errors, review: null, healthScan: [] }; } - const repoData = await fetchRequiredJson(`${API_BASE}/repos/${owner}/${repo}`, { + const repoData = await fetchRequiredJson(`https://api.github.com/repos/${owner}/${repo}`, { authToken, source: 'repository', onError: (source, code, message, details = {}) => addError(errors, source, code, message, details), @@ -706,7 +534,7 @@ export async function importGithubMetadata(repoUrl, options = {}) { const defaultBranch = cleanString(repoData.default_branch ?? ''); const releaseData = await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/releases/latest`, + buildGithubReleaseApiUrl(owner, repo), buildGithubRequestConfig({ authToken, source: 'release', @@ -729,7 +557,7 @@ export async function importGithubMetadata(repoUrl, options = {}) { if (inspectRepositoryFiles) { const branchInfo = defaultBranch ? await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(defaultBranch)}`, + buildGithubBranchApiUrl(owner, repo, defaultBranch), buildGithubRequestConfig({ authToken, source: 'branch', @@ -827,7 +655,6 @@ export async function importGithubMetadata(repoUrl, options = {}) { cleanString, normalizeAuthor, normalizeAuthors, - dedupeAuthors, addWarning, fetchOptionalJson, extractOrcidFromGithubProfile, diff --git a/src/services/githubImporterContributors.js b/src/services/githubImporterContributors.js index 83a7d95..6000a3b 100644 --- a/src/services/githubImporterContributors.js +++ b/src/services/githubImporterContributors.js @@ -1,6 +1,11 @@ -import { buildGithubRequestConfig } from './githubApi.js'; +import { + buildGithubContributorsApiUrl, + buildGithubRequestConfig, + buildGithubUserApiUrl, + buildGithubUserSocialAccountsApiUrl, +} from './githubApi.js'; +import { dedupeAuthors } from './githubImporterAuthors.js'; -const API_BASE = 'https://api.github.com'; const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; const GITHUB_PAGE_SIZE = 100; @@ -55,7 +60,7 @@ async function fetchAllContributors(owner, repo, warnings, authToken, maxContrib while (true) { const pageContributors = await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, + buildGithubContributorsApiUrl(owner, repo, page, GITHUB_PAGE_SIZE), buildGithubRequestConfig({ authToken, source: 'contributors', @@ -112,7 +117,6 @@ export async function fetchContributorAuthors({ cleanString, normalizeAuthor, normalizeAuthors, - dedupeAuthors, addWarning, fetchOptionalJson, extractOrcidFromGithubProfile, @@ -157,7 +161,7 @@ export async function fetchContributorAuthors({ } const profile = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}`, + buildGithubUserApiUrl(login), buildGithubRequestConfig({ authToken, source: 'contributor-profile', @@ -167,7 +171,7 @@ export async function fetchContributorAuthors({ ); const socialAccounts = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, + buildGithubUserSocialAccountsApiUrl(login), buildGithubRequestConfig({ authToken, source: 'contributor-profile-links', diff --git a/src/services/githubImporterMerge.js b/src/services/githubImporterMerge.js new file mode 100644 index 0000000..3962de2 --- /dev/null +++ b/src/services/githubImporterMerge.js @@ -0,0 +1,159 @@ +import { createMetadata } from '../core/metadataModel.js'; +import { + cleanString, + normalizeAuthors, + normalizeGrants, + normalizeKeywords, + normalizeReferences, + normalizeRepoUrl, + normalizeVersionForCompare, +} from './githubImporterUtils.js'; +import { + dedupeAuthors, + enrichAuthorsWithContributorData, + orderAuthorsByContributorRank, +} from './githubImporterAuthors.js'; + +function firstNonEmpty(...values) { + for (const value of values) { + if (Array.isArray(value)) { + if (value.length > 0) { + return value; + } + continue; + } + + const text = cleanString(value); + if (text) { + return text; + } + } + + return ''; +} + +function addWarning(warnings, source, code, message) { + warnings.push({ kind: 'warning', source, code, message }); +} + +function mapTypeOfWork(value) { + const type = cleanString(value).toLowerCase(); + + if (type === 'dataset') { + return 'dataset'; + } + + if (['article', 'book', 'book-chapter', 'conference-paper', 'journal-article', 'manuscript', 'preprint', 'report', 'thesis'].includes(type)) { + return 'article'; + } + + if (type === 'other') { + return 'other'; + } + + return 'software'; +} + +export function mergeMetadata({ + repo, + release, + defaultPublicationDate, + citation, + zenodo, + packageMeta, + readme, + contributors, + contributorLookupAuthors, + supplementalCitationAuthors = [], +}) { + const primaryAuthors = [ + ...normalizeAuthors(Array.isArray(citation?.authors) ? citation.authors : []), + ...normalizeAuthors(Array.isArray(zenodo?.authors) ? zenodo.authors : []), + ...normalizeAuthors(Array.isArray(packageMeta?.authors) ? packageMeta.authors : []), + ...normalizeAuthors(Array.isArray(supplementalCitationAuthors) ? supplementalCitationAuthors : []), + ]; + const authors = [ + ...normalizeAuthors(primaryAuthors), + ...normalizeAuthors(Array.isArray(contributors) ? contributors : []), + ]; + const keywords = normalizeKeywords(firstNonEmpty(citation?.keywords, zenodo?.keywords, packageMeta?.keywords, repo?.topics)); + const references = normalizeReferences(firstNonEmpty(zenodo?.references, citation?.references)); + const grants = normalizeGrants(firstNonEmpty(zenodo?.grants)); + const enrichedAuthors = enrichAuthorsWithContributorData(authors, contributorLookupAuthors); + const dedupedAuthors = dedupeAuthors(enrichedAuthors); + const orderedAuthors = orderAuthorsByContributorRank(dedupedAuthors, contributorLookupAuthors); + + return createMetadata({ + title: cleanString(firstNonEmpty(citation?.title, zenodo?.title, packageMeta?.title, repo?.name)), + authors: orderedAuthors, + keywords, + license: cleanString(firstNonEmpty(citation?.license, zenodo?.license, packageMeta?.license, repo?.license?.spdx_id)), + typeOfWork: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), + customTypeOfWork: '', + zenodoUploadType: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), + version: cleanString(firstNonEmpty(citation?.version, zenodo?.version, packageMeta?.version, release?.tag_name)), + publicationDate: cleanString(firstNonEmpty(release?.published_at, citation?.publicationDate, zenodo?.publicationDate, defaultPublicationDate)).split('T')[0], + repositoryCode: normalizeRepoUrl(firstNonEmpty(repo?.html_url, citation?.repositoryCode, packageMeta?.repositoryCode)), + doi: cleanString(firstNonEmpty(zenodo?.doi, citation?.doi)), + abstract: cleanString(firstNonEmpty(citation?.abstract, zenodo?.abstract, packageMeta?.abstract, readme, repo?.description)), + references, + grants, + }); +} + +export function addCitationConsistencyWarnings({ warnings, citation, zenodo, releaseData, repoData, metadata }) { + const releaseTag = cleanString(releaseData?.tag_name ?? ''); + const citationVersion = cleanString(citation?.version ?? ''); + const zenodoVersion = cleanString(zenodo?.version ?? ''); + const finalVersion = cleanString(metadata?.version ?? ''); + const normalizedReleaseTag = normalizeVersionForCompare(releaseTag); + const normalizedCitationVersion = normalizeVersionForCompare(citationVersion); + const normalizedZenodoVersion = normalizeVersionForCompare(zenodoVersion); + + if (normalizedReleaseTag && normalizedCitationVersion && normalizedReleaseTag !== normalizedCitationVersion) { + addWarning(warnings, 'citation', 'version-mismatch', `CITATION.cff version (${citationVersion}) differs from latest release tag (${releaseTag}); using CITATION.cff version for import.`); + } + + if (normalizedReleaseTag && normalizedZenodoVersion && normalizedReleaseTag !== normalizedZenodoVersion) { + addWarning(warnings, 'zenodo', 'version-mismatch', `.zenodo.json version (${zenodoVersion}) differs from latest release tag (${releaseTag}); using .zenodo.json version for import.`); + } + + if (normalizedCitationVersion && normalizedZenodoVersion && normalizedCitationVersion !== normalizedZenodoVersion) { + addWarning(warnings, 'citation', 'cross-file-version-mismatch', `CITATION.cff version (${citationVersion}) and .zenodo.json version (${zenodoVersion}) differ.`); + } + + const citationDate = cleanString(citation?.publicationDate ?? '').split('T')[0]; + const zenodoDate = cleanString(zenodo?.publicationDate ?? '').split('T')[0]; + const releaseDate = cleanString(releaseData?.published_at ?? '').split('T')[0]; + + if (releaseDate && citationDate && releaseDate !== citationDate) { + addWarning(warnings, 'citation', 'date-mismatch', `CITATION.cff date-released (${citationDate}) differs from latest release date (${releaseDate}); using release date for import.`); + } + + if (releaseDate && zenodoDate && releaseDate !== zenodoDate) { + addWarning(warnings, 'zenodo', 'date-mismatch', `.zenodo.json publication_date (${zenodoDate}) differs from latest release date (${releaseDate}); using release date for import.`); + } + + const repoUrl = normalizeRepoUrl(repoData?.html_url ?? ''); + const citationRepoUrl = normalizeRepoUrl(citation?.repositoryCode ?? ''); + + if (repoUrl && citationRepoUrl && repoUrl !== citationRepoUrl) { + addWarning(warnings, 'citation', 'repository-url-mismatch', `CITATION.cff repository-code (${citationRepoUrl}) differs from repository URL (${repoUrl}); using repository URL for import.`); + } + + if (!finalVersion) { + addWarning(warnings, 'citation', 'missing-version', 'No version could be determined from release tag, CITATION.cff, .zenodo.json, or package metadata.'); + } + + const repoSpdx = cleanString(repoData?.license?.spdx_id ?? '').toUpperCase(); + const citationLicense = cleanString(citation?.license ?? '').toUpperCase(); + const zenodoLicense = cleanString(zenodo?.license ?? '').toUpperCase(); + + if (repoSpdx && citationLicense && repoSpdx !== citationLicense) { + addWarning(warnings, 'citation', 'license-mismatch', `CITATION.cff license (${citationLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`); + } + + if (repoSpdx && zenodoLicense && repoSpdx !== zenodoLicense) { + addWarning(warnings, 'zenodo', 'license-mismatch', `.zenodo.json license (${zenodoLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`); + } +} From 39c463940edc20152ebd0db8b791d24adc9966e3 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:38:38 -0400 Subject: [PATCH 05/23] refactor: enhance stripWrappingQuotes function to handle mismatched quotes --- src/services/githubImporterUtils.js | 9 ++++++++- tests/services/githubImporter.test.js | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/services/githubImporterUtils.js b/src/services/githubImporterUtils.js index 42a0c6e..947cbd7 100644 --- a/src/services/githubImporterUtils.js +++ b/src/services/githubImporterUtils.js @@ -10,7 +10,14 @@ function stripWrappingQuotes(value) { return ''; } - return text.replace(/^"|"$/g, '').replace(/^'|'$/g, ''); + const first = text[0]; + const last = text[text.length - 1]; + + if ((first === '"' && last === '"') || (first === '\'' && last === '\'')) { + return text.slice(1, -1); + } + + return text; } function firstNonEmpty(...values) { diff --git a/tests/services/githubImporter.test.js b/tests/services/githubImporter.test.js index 1a63d0f..6397234 100644 --- a/tests/services/githubImporter.test.js +++ b/tests/services/githubImporter.test.js @@ -59,6 +59,7 @@ test('stripWrappingQuotes removes matching quote wrappers without altering inner assert.equal(stripWrappingQuotes("'OpenCite'"), 'OpenCite'); assert.equal(stripWrappingQuotes('OpenCite'), 'OpenCite'); assert.equal(stripWrappingQuotes('"quoted \\"text\\""'), 'quoted \\"text\\"'); + assert.equal(stripWrappingQuotes('"OpenCite\''), '"OpenCite\''); }); test('parseCitationCff emits warning for preferred-citation sections', () => { From 0d3e9795257b22644db856429adaec267ea2b849 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:40:42 -0400 Subject: [PATCH 06/23] refactor: remove redundant firstNonEmpty function and import it from core utilities --- src/services/githubImporterMerge.js | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/services/githubImporterMerge.js b/src/services/githubImporterMerge.js index 3962de2..d32f156 100644 --- a/src/services/githubImporterMerge.js +++ b/src/services/githubImporterMerge.js @@ -1,6 +1,7 @@ import { createMetadata } from '../core/metadataModel.js'; import { cleanString, + firstNonEmpty, normalizeAuthors, normalizeGrants, normalizeKeywords, @@ -14,24 +15,6 @@ import { orderAuthorsByContributorRank, } from './githubImporterAuthors.js'; -function firstNonEmpty(...values) { - for (const value of values) { - if (Array.isArray(value)) { - if (value.length > 0) { - return value; - } - continue; - } - - const text = cleanString(value); - if (text) { - return text; - } - } - - return ''; -} - function addWarning(warnings, source, code, message) { warnings.push({ kind: 'warning', source, code, message }); } From 299f4b7b6906a8dab9d9ea88efe3c3053ca58bb6 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:46:52 -0400 Subject: [PATCH 07/23] refactor: fix export statement for addCitationConsistencyWarnings and mergeMetadata --- src/services/githubImporter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 0394c0d..dce04bf 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -31,7 +31,7 @@ import { resolveContributorFallbackLimit, } from './githubImporterContributors.js'; import { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; -export { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; +export { addCitationConsistencyWarnings, mergeMetadata }; import { parseCargoToml, parsePackageJson, From 123b897088cf97e9a70c9250d1dccf4fe2cf7eff Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:49:01 -0400 Subject: [PATCH 08/23] refactor: rename parseJsonSafely to parseJson for consistency --- src/services/githubImporter.js | 4 ++-- src/services/githubImporterParsers.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index dce04bf..be9a29d 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -414,12 +414,12 @@ export function parseCitationCff(text) { }; } -function parseJsonSafely(text) { +function parseJson(text) { return JSON.parse(text); } export function parseZenodoJson(text) { - const payload = parseJsonSafely(text); + const payload = parseJson(text); const creators = Array.isArray(payload.creators) ? payload.creators.map((creator) => normalizeAuthor(creator)).filter(Boolean) : []; diff --git a/src/services/githubImporterParsers.js b/src/services/githubImporterParsers.js index a985795..df8ff0d 100644 --- a/src/services/githubImporterParsers.js +++ b/src/services/githubImporterParsers.js @@ -12,7 +12,7 @@ const normalizeKeywords = utilNormalizeKeywords; const normalizeRepoUrl = utilNormalizeRepoUrl; const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; -function parseJsonSafely(text) { +function parseJson(text) { return JSON.parse(text); } @@ -31,7 +31,7 @@ function extractPackageAuthors(payload) { } export function parsePackageJson(text) { - const payload = parseJsonSafely(text); + const payload = parseJson(text); return { title: cleanString(payload.name ?? ''), From 3af823bd05d107075ac9ca12736fe7136e53f8aa Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:57:55 -0400 Subject: [PATCH 09/23] refactor: simplify rate limit hint message for GitHub authentication --- src/services/githubApi.js | 5 ----- src/services/githubImporter.js | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/services/githubApi.js b/src/services/githubApi.js index 3274196..e4e6fb4 100644 --- a/src/services/githubApi.js +++ b/src/services/githubApi.js @@ -73,11 +73,6 @@ export function resolveGithubToken(options = {}) { return explicitToken; } - const envToken = cleanString(import.meta.env?.VITE_GITHUB_TOKEN ?? ''); - if (envToken) { - return envToken; - } - try { if (typeof window !== 'undefined' && window.localStorage) { const localToken = cleanString(window.localStorage.getItem('opencite_github_token') ?? ''); diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index be9a29d..13a57c9 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -90,7 +90,7 @@ function addRateLimitHintIfNeeded(warnings, authToken) { warnings, 'github-auth', 'rate-limit-hint', - 'To reduce rate limits, set VITE_GITHUB_TOKEN in .env.local or set localStorage.opencite_github_token.', + 'To reduce rate limits, pass authToken explicitly or set localStorage.opencite_github_token.', ); } } From 6281153fdc516cbd9a96ffca274fae3cb1999c7a Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Sat, 8 Aug 2026 01:00:32 -0400 Subject: [PATCH 10/23] refactor: modularize GitHub metadata import pipeline Split importer responsibilities into focused modules for author matching, contributor retrieval, file parsing, and shared normalization utilities. Centralized author and repository normalization logic to reduce duplication and improve maintainability. Updated citation validation tests to verify required author-field handling remains correct after the refactor. --- src/services/githubImporter.js | 981 ++------------------- src/services/githubImporterAuthors.js | 269 ++++++ src/services/githubImporterContributors.js | 258 ++++++ src/services/githubImporterParsers.js | 170 ++++ src/services/githubImporterUtils.js | 271 ++++++ tests/services/citationValidation.test.js | 10 +- 6 files changed, 1029 insertions(+), 930 deletions(-) create mode 100644 src/services/githubImporterAuthors.js create mode 100644 src/services/githubImporterContributors.js create mode 100644 src/services/githubImporterParsers.js create mode 100644 src/services/githubImporterUtils.js diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index a1ac604..2961817 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -1,5 +1,5 @@ import { createMetadata } from '../core/metadataModel.js'; -import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile, normalizeOrcid } from '../utils/orcid.js'; +import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile } from '../utils/orcid.js'; import { validateCitationCffText } from './citationValidation.js'; import { runCitationHealthScan } from './citationHealthScan.js'; import { @@ -10,14 +10,40 @@ import { parseGithubUrl, resolveGithubToken, } from './githubApi.js'; +import { + cleanString as utilCleanString, + extractFirstMarkdownParagraph as utilExtractFirstMarkdownParagraph, + firstNonEmpty as utilFirstNonEmpty, + normalizeAuthor as utilNormalizeAuthor, + normalizeAuthors as utilNormalizeAuthors, + normalizeGrants as utilNormalizeGrants, + normalizeKeywords as utilNormalizeKeywords, + normalizeReferences as utilNormalizeReferences, + normalizeRepoUrl as utilNormalizeRepoUrl, + normalizeVersionForCompare as utilNormalizeVersionForCompare, +} from './githubImporterUtils.js'; +import { + fetchContributorAuthors, + resolveContributorFallbackLimit, +} from './githubImporterContributors.js'; +import { + dedupeAuthors, + enrichAuthorsWithContributorData, + orderAuthorsByContributorRank, +} from './githubImporterAuthors.js'; +import { + parseCargoToml, + parsePackageJson, + parsePomXml, + parsePyprojectToml, + parseReadme, + parseSetupPy, +} from './githubImporterParsers.js'; import { compareExistingMetadataFiles } from './metadataComparison.js'; import { runMetadataReviewPipeline } from './metadataReview.js'; import { validateZenodoJsonText } from './zenodoValidation.js'; const API_BASE = 'https://api.github.com'; -const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; -const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; -const GITHUB_PAGE_SIZE = 100; const FILES_TO_INSPECT = [ 'CITATION.cff', '.zenodo.json', @@ -29,6 +55,17 @@ const FILES_TO_INSPECT = [ 'pom.xml', ]; +const cleanString = utilCleanString; +const firstNonEmpty = utilFirstNonEmpty; +const normalizeKeywords = utilNormalizeKeywords; +const normalizeReferences = utilNormalizeReferences; +const normalizeGrants = utilNormalizeGrants; +const normalizeAuthor = utilNormalizeAuthor; +const normalizeAuthors = utilNormalizeAuthors; +const normalizeRepoUrl = utilNormalizeRepoUrl; +const normalizeVersionForCompare = utilNormalizeVersionForCompare; +const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; + function makeIssue(kind, source, code, message, details = {}) { return { kind, source, code, message, ...details }; } @@ -41,234 +78,6 @@ function addError(errors, source, code, message, details = {}) { errors.push(makeIssue('error', source, code, message, details)); } -function cleanString(value) { - return String(value ?? '').replace(/[\t ]+/g, ' ').trim(); -} - -function firstNonEmpty(...values) { - for (const value of values) { - if (Array.isArray(value)) { - if (value.length > 0) { - return value; - } - continue; - } - - const text = cleanString(value); - if (text) { - return text; - } - } - - return ''; -} - -function normalizeStringList(value) { - if (Array.isArray(value)) { - return value.map((item) => cleanString(item)).filter(Boolean); - } - - if (!value) { - return []; - } - - return String(value) - .split(/[\n,]/) - .map((item) => cleanString(item)) - .filter(Boolean); -} - -function normalizeKeywords(value) { - return [...new Set(normalizeStringList(value).map((keyword) => keyword.toLowerCase()))]; -} - -function normalizeReferences(value) { - if (Array.isArray(value)) { - return value.map((item) => cleanString(item)).filter(Boolean); - } - - if (!value) { - return []; - } - - return String(value) - .split(/\n+/) - .map((item) => cleanString(item)) - .filter(Boolean); -} - -function normalizeGrants(value) { - if (Array.isArray(value)) { - return value - .map((item) => { - if (typeof item === 'string') { - return cleanString(item); - } - - if (item && typeof item === 'object') { - return cleanString(item.id ?? item.value ?? item.grantId ?? ''); - } - - return ''; - }) - .filter(Boolean); - } - - if (!value) { - return []; - } - - return String(value) - .split(/\n+/) - .map((item) => cleanString(item)) - .filter(Boolean); -} - -function capitalizeToken(token) { - const text = cleanString(token); - if (!text) { - return ''; - } - - return text - .split(/([\-'])/) - .map((part) => { - if (part === '-' || part === "'") { - return part; - } - - // Preserve mixed-case tokens (for example, McDonald) and normalize others. - if (/[a-z]/.test(part) && /[A-Z]/.test(part)) { - return part; - } - - const lower = part.toLowerCase(); - return lower.charAt(0).toUpperCase() + lower.slice(1); - }) - .join(''); -} - -function capitalizeName(value) { - return cleanString(value) - .split(/\s+/) - .map((part) => capitalizeToken(part)) - .filter(Boolean) - .join(' '); -} - -function humanizeIdentifier(value) { - return cleanString(value) - .replace(/[._-]+/g, ' ') - .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') - .replace(/([a-z\d])([A-Z])/g, '$1 $2') - .replace(/([A-Za-z])(\d)/g, '$1 $2') - .replace(/(\d)([A-Za-z])/g, '$1 $2'); -} - -function splitDisplayName(name) { - const value = cleanString(name); - - if (!value) { - return { givenNames: '', familyNames: '' }; - } - - if (value.includes(',')) { - const [familyNames, ...givenParts] = value.split(','); - return { - givenNames: capitalizeName(givenParts.join(',').trim()), - familyNames: capitalizeName(familyNames), - }; - } - - const normalized = humanizeIdentifier(value); - const parts = normalized.split(/\s+/).filter(Boolean); - - if (parts.length <= 1) { - return { givenNames: capitalizeName(parts[0] ?? ''), familyNames: '' }; - } - - return { - givenNames: capitalizeName(parts.slice(0, -1).join(' ')), - familyNames: capitalizeName(parts[parts.length - 1]), - }; -} - -function normalizeAuthor(input) { - if (!input) { - return null; - } - - if (typeof input === 'string') { - const { givenNames, familyNames } = splitDisplayName(input); - return givenNames || familyNames ? { givenNames, familyNames, orcid: '', affiliation: '' } : null; - } - - if (typeof input !== 'object') { - return null; - } - - const name = cleanString(input.name ?? input.fullName ?? input.full_name ?? input.creator_name ?? ''); - const parsedName = name ? splitDisplayName(name) : null; - let givenNames = capitalizeName(input.givenNames ?? input['given-names'] ?? input.firstName ?? input.firstname ?? parsedName?.givenNames ?? ''); - let familyNames = capitalizeName(input.familyNames ?? input['family-names'] ?? input.lastName ?? input.lastname ?? parsedName?.familyNames ?? ''); - const affiliation = cleanString(input.affiliation ?? input.organization ?? input.company ?? input.institution ?? ''); - const orcid = normalizeOrcid(input.orcid ?? input.ORCID ?? input.orcidId ?? ''); - - // Some sources put full names in a single first-name field without spaces. - if (givenNames && !familyNames) { - const reparsed = splitDisplayName(givenNames); - if (reparsed.familyNames) { - givenNames = reparsed.givenNames; - familyNames = reparsed.familyNames; - } - } - - if (!givenNames && !familyNames && !affiliation && !orcid) { - return null; - } - - return { givenNames, familyNames, orcid, affiliation }; -} - -function normalizeAuthors(value) { - if (!Array.isArray(value)) { - return []; - } - - return value.map((item) => normalizeAuthor(item)).filter(Boolean); -} - -function normalizeRepoUrl(value) { - const text = cleanString(value); - if (!text) { - return ''; - } - - const trimmed = text.replace(/^git\+/, '').replace(/\.git$/i, '').replace(/\/+$/, ''); - - try { - const parsed = new URL(trimmed); - const host = parsed.hostname.toLowerCase(); - let pathname = parsed.pathname.replace(/\/+$/, ''); - if (host === 'github.com') { - pathname = pathname.toLowerCase(); - } - return `${parsed.protocol}//${host}${pathname}`; - } catch { - return trimmed; - } -} - -function normalizeVersionForCompare(value) { - const text = cleanString(value).toLowerCase(); - if (!text) { - return ''; - } - - // Treat v-prefixed tags and bare semver as equivalent for mismatch checks. - return text.replace(/^v(?=\d)/, ''); -} - function addRateLimitHintIfNeeded(warnings, authToken) { if (authToken) { return; @@ -287,30 +96,6 @@ function addRateLimitHintIfNeeded(warnings, authToken) { } } -async function fetchOrcidFromGithubProfileHtml(profileUrl) { - const url = cleanString(profileUrl); - if (!url) { - return null; - } - - try { - const response = await fetch(url, { - headers: { - Accept: 'text/html', - }, - }); - - if (!response.ok) { - return null; - } - - const html = await response.text(); - return extractOrcidFromGithubHtml(html); - } catch { - return null; - } -} - function shouldInspectRepositoryFiles(options = {}) { return options.inspectRepositoryFiles !== false; } @@ -394,90 +179,6 @@ export function summarizeImportedMetadataFiles(fileContents = {}) { return summary; } -function resolveContributorFallbackLimit(options = {}) { - if (!Object.prototype.hasOwnProperty.call(options, 'contributorFallbackLimit')) { - return TOP_CONTRIBUTOR_FALLBACK_LIMIT; - } - - if (options.contributorFallbackLimit == null || options.contributorFallbackLimit === '') { - return null; - } - - const rawLimit = Number(options.contributorFallbackLimit); - - if (!Number.isFinite(rawLimit)) { - return TOP_CONTRIBUTOR_FALLBACK_LIMIT; - } - - return Math.min(Math.max(Math.trunc(rawLimit), 1), MAX_CONTRIBUTOR_FALLBACK_LIMIT); -} - -async function fetchAllContributors(owner, repo, warnings, authToken = '', maxContributors = null) { - const contributors = []; - let page = 1; - - while (true) { - const pageContributors = await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, - { - authToken, - source: 'contributors', - label: `contributors page ${page}`, - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, - ) || []; - - if (!Array.isArray(pageContributors) || pageContributors.length === 0) { - break; - } - - contributors.push(...pageContributors); - - if (maxContributors && contributors.length >= maxContributors) { - return contributors.slice(0, maxContributors); - } - - if (pageContributors.length < GITHUB_PAGE_SIZE) { - break; - } - - page += 1; - } - - return contributors; -} - -function extractFirstMarkdownParagraph(text) { - const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); - const paragraph = []; - let started = false; - - for (const line of lines) { - const trimmed = line.trim(); - - if (!trimmed) { - if (started) { - break; - } - continue; - } - - if (!started && /^#{1,6}\s+/.test(trimmed)) { - started = true; - continue; - } - - if (!started && /^(!|\[|-)/.test(trimmed)) { - continue; - } - - started = true; - paragraph.push(trimmed); - } - - return paragraph.join(' ').replace(/\s+/g, ' ').trim(); -} - export function parseCitationCff(text) { const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); const result = { @@ -763,159 +464,6 @@ export function parseZenodoJson(text) { }; } -function extractPackageAuthors(payload) { - const candidates = []; - - if (payload.author) { - candidates.push(payload.author); - } - - if (Array.isArray(payload.authors)) { - candidates.push(...payload.authors); - } - - return candidates.map((item) => normalizeAuthor(item)).filter(Boolean); -} - -function parsePackageJson(text) { - const payload = parseJsonSafely(text); - - return { - title: cleanString(payload.name ?? ''), - abstract: cleanString(payload.description ?? ''), - version: cleanString(payload.version ?? ''), - repositoryCode: normalizeRepoUrl(typeof payload.repository === 'string' ? payload.repository : payload.repository?.url ?? payload.homepage ?? ''), - license: cleanString(typeof payload.license === 'string' ? payload.license : payload.license?.type ?? ''), - keywords: normalizeKeywords(payload.keywords), - authors: extractPackageAuthors(payload), - }; -} - -function extractTomlSection(text, sectionName) { - const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); - const sectionLines = []; - let inSection = false; - - for (const line of lines) { - const sectionMatch = line.trim().match(/^\[([^\]]+)\]$/); - - if (sectionMatch) { - if (inSection) { - break; - } - - inSection = sectionMatch[1] === sectionName; - continue; - } - - if (inSection) { - sectionLines.push(line); - } - } - - return sectionLines.join('\n'); -} - -function extractTomlValue(sectionText, key) { - const match = sectionText.match(new RegExp(`^${key}\\s*=\\s*(.+)$`, 'm')); - return match ? match[1].trim() : ''; -} - -function parseTomlString(sectionText, key) { - const value = extractTomlValue(sectionText, key); - const match = value.match(/^['"](.+?)['"]$/); - return match ? match[1].trim() : ''; -} - -function parseTomlStrings(value) { - return [...String(value ?? '').matchAll(/['"]([^'"]+)['"]/g)].map((match) => cleanString(match[1])).filter(Boolean); -} - -function parsePyprojectToml(text) { - const section = extractTomlSection(text, 'project') || extractTomlSection(text, 'tool.poetry'); - const authorsBlock = extractTomlValue(section, 'authors') || extractTomlValue(section, 'maintainers'); - const authors = [...String(authorsBlock ?? '').matchAll(/name\s*=\s*['"]([^'"]+)['"]/g)] - .map((match) => normalizeAuthor({ name: match[1] })) - .filter(Boolean); - - const licenseValue = parseTomlString(section, 'license') || cleanString((section.match(/license\s*=\s*\{[^}]*text\s*=\s*['"]([^'"]+)['"][^}]*\}/s) || [])[1] ?? ''); - const repositoryCode = parseTomlString(section, 'repository') || parseTomlString(section, 'homepage') || parseTomlString(section, 'url'); - - return { - title: parseTomlString(section, 'name'), - abstract: parseTomlString(section, 'description'), - version: parseTomlString(section, 'version'), - repositoryCode, - license: licenseValue, - keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), - authors, - }; -} - -function parseSetupPy(text) { - const source = String(text ?? ''); - const extract = (key) => cleanString((source.match(new RegExp(`${key}\\s*=\\s*['"]([^'"]+)['"]`, 'm')) || [])[1] ?? ''); - - const authors = []; - const author = extract('author'); - const maintainer = extract('maintainer'); - - if (author) { - authors.push(normalizeAuthor({ name: author })); - } else if (maintainer) { - authors.push(normalizeAuthor({ name: maintainer })); - } - - return { - title: extract('name'), - abstract: extract('description'), - version: extract('version'), - repositoryCode: normalizeRepoUrl(extract('url')), - license: extract('license'), - keywords: normalizeKeywords(extract('keywords')), - authors: authors.filter(Boolean), - }; -} - -function parseCargoToml(text) { - const section = extractTomlSection(text, 'package'); - const authors = parseTomlStrings(extractTomlValue(section, 'authors')).map((name) => normalizeAuthor({ name })).filter(Boolean); - - return { - title: parseTomlString(section, 'name'), - abstract: parseTomlString(section, 'description'), - version: parseTomlString(section, 'version'), - repositoryCode: parseTomlString(section, 'repository'), - license: parseTomlString(section, 'license'), - keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), - authors, - }; -} - -function parsePomXml(text) { - const source = String(text ?? ''); - const extract = (pattern) => cleanString((source.match(pattern) || [])[1] ?? ''); - const authors = [...source.matchAll(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/developer>/g)] - .map((match) => normalizeAuthor({ name: match[1] })) - .filter(Boolean); - - const licenseMatch = source.match(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/license>/); - - return { - title: extract(/([^<]+)<\/name>/), - abstract: extract(/([^<]+)<\/description>/), - version: extract(/([^<]+)<\/version>/), - repositoryCode: extract(/([^<]+)<\/url>/), - license: cleanString((licenseMatch || [])[1] ?? ''), - keywords: [], - authors, - }; -} - -function parseReadme(text) { - return extractFirstMarkdownParagraph(text); -} - function parseFile(path, text, warnings, errors) { try { if (path === '.zenodo.json') { @@ -983,123 +531,6 @@ function mapTypeOfWork(value) { return 'software'; } -function authorNameKey(author) { - return [ - cleanString(author?.givenNames ?? '').toLowerCase(), - cleanString(author?.familyNames ?? '').toLowerCase(), - ].join('|'); -} - -function normalizeNameToken(value) { - return cleanString(value) - .toLowerCase() - .replace(/[^a-z0-9\s]/g, ' ') - .replace(/\s+/g, ' ') - .trim(); -} - -function authorMatchMetadata(author) { - const givenNames = normalizeNameToken(author?.givenNames ?? ''); - const familyNames = normalizeNameToken(author?.familyNames ?? ''); - const givenTokens = givenNames.split(' ').filter(Boolean); - const familyTokens = familyNames.split(' ').filter(Boolean); - - return { - givenNames, - familyNames, - givenFirst: givenTokens[0] ?? '', - givenInitials: givenTokens.map((token) => token[0]).join(''), - familyLast: familyTokens[familyTokens.length - 1] ?? '', - fullName: [givenNames, familyNames].filter(Boolean).join(' ').trim(), - }; -} - -function authorAltNameKeys(author) { - const metadata = authorMatchMetadata(author); - const givenNames = metadata.givenNames; - const familyNames = metadata.familyNames; - const givenFirst = metadata.givenFirst; - const familyLast = metadata.familyLast; - const keys = new Set([ - `${givenNames}|${familyNames}`, - `${givenFirst}|${familyNames}`, - `${givenNames}|${familyLast}`, - `${givenFirst}|${familyLast}`, - ]); - - keys.delete('|'); - keys.delete(''); - return [...keys].filter(Boolean); -} - -function authorsLikelyMatch(sourceAuthor, contributorAuthor) { - const source = authorMatchMetadata(sourceAuthor); - const contributor = authorMatchMetadata(contributorAuthor); - - if (!source.familyLast || !contributor.familyLast || source.familyLast !== contributor.familyLast) { - return false; - } - - if (source.givenNames && contributor.givenNames && source.givenNames === contributor.givenNames) { - return true; - } - - if (source.givenFirst && contributor.givenFirst && source.givenFirst === contributor.givenFirst) { - return true; - } - - if (source.givenInitials && contributor.givenInitials && source.givenInitials === contributor.givenInitials) { - return true; - } - - if (source.fullName && contributor.fullName && source.fullName === contributor.fullName) { - return true; - } - - return false; -} - -function enrichAuthorsWithContributorData(sourceAuthors, contributorAuthors) { - const normalizedSourceAuthors = normalizeAuthors(Array.isArray(sourceAuthors) ? sourceAuthors : []); - if (normalizedSourceAuthors.length === 0) { - return normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); - } - - const normalizedContributorAuthors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); - const contributorMatches = new Map(); - - for (const contributorAuthor of normalizedContributorAuthors) { - if (!contributorAuthor.orcid && !contributorAuthor.affiliation) { - continue; - } - - for (const key of authorAltNameKeys(contributorAuthor)) { - const existing = contributorMatches.get(key) ?? []; - existing.push(contributorAuthor); - contributorMatches.set(key, existing); - } - } - - return normalizedSourceAuthors.map((author) => { - const keyedMatches = authorAltNameKeys(author) - .flatMap((key) => contributorMatches.get(key) ?? []); - const heuristicMatches = normalizedContributorAuthors.filter((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); - const matches = [...new Set([...keyedMatches, ...heuristicMatches])]; - const uniqueOrcids = [...new Set(matches.map((match) => match.orcid).filter(Boolean))]; - const uniqueAffiliations = [...new Set(matches.map((match) => match.affiliation).filter(Boolean))]; - - if ((!author.orcid && uniqueOrcids.length > 1) || (!author.affiliation && uniqueAffiliations.length > 1)) { - return author; - } - - return { - ...author, - orcid: author.orcid || uniqueOrcids[0] || '', - affiliation: author.affiliation || uniqueAffiliations[0] || '', - }; - }); -} - function mergeMetadata({ repo, release, @@ -1384,14 +815,22 @@ export async function importGithubMetadata(repoUrl, options = {}) { const readme = parsedFiles['readme.md']?.abstract || ''; const hasPrimaryAuthors = firstNonEmpty(citation?.authors, zenodo?.authors, packageMeta?.authors, supplementalCitationAuthors).length > 0; - const contributorResult = await fetchContributorAuthors( + const contributorResult = await fetchContributorAuthors({ owner, repo, warnings, authToken, contributorFallbackLimit, - !hasPrimaryAuthors, - ); + emitFallbackWarning: !hasPrimaryAuthors, + cleanString, + normalizeAuthor, + normalizeAuthors, + dedupeAuthors, + addWarning, + fetchOptionalJson, + extractOrcidFromGithubProfile, + extractOrcidFromGithubHtml, + }); const contributors = contributorResult.fallbackAuthors.filter(Boolean); const contributorLookupAuthors = contributorResult.lookupAuthors.filter(Boolean); @@ -1471,319 +910,3 @@ export async function importGithubMetadata(repoUrl, options = {}) { return { metadata, warnings, errors, review, healthScan, comparisons }; } -async function fetchContributorAuthors( - owner, - repo, - warnings, - authToken = '', - contributorFallbackLimit = TOP_CONTRIBUTOR_FALLBACK_LIMIT, - emitFallbackWarning = true, -) { - const contributors = await fetchAllContributors(owner, repo, warnings, authToken, contributorFallbackLimit); - - if (!Array.isArray(contributors) || contributors.length === 0) { - return { - fallbackAuthors: [], - lookupAuthors: [], - }; - } - - if (emitFallbackWarning) { - addWarning( - warnings, - 'authors', - 'commit-based-fallback', - contributorFallbackLimit - ? `Using top ${contributorFallbackLimit} contributors as fallback authors.` - : 'Using contributors as fallback authors.', - { owner, repo }, - ); - } - - const profiles = await Promise.all( - contributors.map(async (contributor) => { - const login = cleanString(contributor?.login ?? ''); - if (!login) { - return { - contributor, - profile: null, - socialAccounts: [], - author: null, - autoFilledOrcid: false, - excludedAutomated: false, - }; - } - - const profile = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}`, - { - authToken, - source: 'contributor-profile', - label: `the profile for ${login}`, - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, - ); - - const socialAccounts = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, - { - authToken, - source: 'contributor-profile-links', - label: `the profile links for ${login}`, - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, - ) || []; - - if (isAutomatedContributor(contributor, profile)) { - return { - contributor, - profile, - socialAccounts, - author: null, - autoFilledOrcid: false, - excludedAutomated: true, - }; - } - - let profileOrcid = extractOrcidFromGithubProfile(profile, socialAccounts); - if (!profileOrcid) { - profileOrcid = await fetchOrcidFromGithubProfileHtml(profile?.html_url ?? contributor?.html_url ?? ''); - } - - if (profile?.name) { - return { - contributor, - profile, - socialAccounts, - author: normalizeAuthor({ - name: profile.name, - affiliation: profile.company ?? '', - orcid: profileOrcid, - }), - autoFilledOrcid: Boolean(profileOrcid), - excludedAutomated: false, - }; - } - - // Fallback to contributor login when profile name is missing. - return { - contributor, - profile, - socialAccounts, - author: normalizeAuthor({ - name: login, - affiliation: '', - orcid: profileOrcid, - }), - autoFilledOrcid: Boolean(profileOrcid), - excludedAutomated: false, - }; - }), - ); - - const excludedAutomatedCount = profiles.filter((entry) => entry?.excludedAutomated).length; - if (excludedAutomatedCount > 0) { - addWarning( - warnings, - 'authors', - 'automated-contributors-excluded', - `Excluded ${excludedAutomatedCount} automated account(s) from fallback authors.`, - { owner, repo }, - ); - } - - const autoFilledOrcidCount = profiles.filter((entry) => entry?.autoFilledOrcid).length; - if (autoFilledOrcidCount > 0) { - addWarning( - warnings, - 'authors', - 'orcid-autofilled', - `Auto-filled ORCID for ${autoFilledOrcidCount} contributor(s) from GitHub profile data.`, - { owner, repo }, - ); - } - - const fallbackAuthors = profiles - .slice(0, contributorFallbackLimit ?? profiles.length) - .map((entry) => entry?.author); - const lookupAuthors = profiles.map((entry) => entry?.author); - - return { - fallbackAuthors: dedupeAuthors(normalizeAuthors(fallbackAuthors)), - lookupAuthors: dedupeAuthors(normalizeAuthors(lookupAuthors)), - }; -} - -function dedupeAuthors(authors) { - const seen = new Set(); - const byOrcid = new Map(); - const byName = new Map(); - const deduped = []; - - for (const rawAuthor of authors) { - const author = normalizeAuthor(rawAuthor); - if (!author) { - continue; - } - - const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); - const nameKey = [ - cleanString(author?.givenNames ?? '').toLowerCase(), - cleanString(author?.familyNames ?? '').toLowerCase(), - ].join('|'); - - if (nameKey !== '|' && byName.has(nameKey)) { - const existing = byName.get(nameKey); - const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); - const canMergeByName = !existingOrcid || !orcidKey || existingOrcid === orcidKey; - - if (canMergeByName) { - if (!existing.orcid && author.orcid) { - existing.orcid = author.orcid; - } - if (!existing.affiliation && author.affiliation) { - existing.affiliation = author.affiliation; - } - if (orcidKey && !byOrcid.has(orcidKey)) { - byOrcid.set(orcidKey, existing); - } - continue; - } - } - - const likelyMatch = deduped.find((existing) => { - const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); - const hasConflictingOrcid = existingOrcid && orcidKey && existingOrcid !== orcidKey; - - if (hasConflictingOrcid) { - return false; - } - - return authorsLikelyMatch(existing, author); - }); - - if (likelyMatch) { - if (!likelyMatch.orcid && author.orcid) { - likelyMatch.orcid = author.orcid; - } - if (!likelyMatch.affiliation && author.affiliation) { - likelyMatch.affiliation = author.affiliation; - } - - const mergedOrcidKey = cleanString(likelyMatch?.orcid ?? '').toLowerCase(); - if (mergedOrcidKey && !byOrcid.has(mergedOrcidKey)) { - byOrcid.set(mergedOrcidKey, likelyMatch); - } - continue; - } - - if (orcidKey && byOrcid.has(orcidKey)) { - const existing = byOrcid.get(orcidKey); - if (!existing.affiliation && author.affiliation) { - existing.affiliation = author.affiliation; - } - continue; - } - - const key = [ - cleanString(author?.givenNames ?? '').toLowerCase(), - cleanString(author?.familyNames ?? '').toLowerCase(), - cleanString(author?.orcid ?? '').toLowerCase(), - ].join('|'); - - if (!key || seen.has(key)) { - continue; - } - - seen.add(key); - const normalizedAuthor = { ...author }; - deduped.push(normalizedAuthor); - - if (orcidKey) { - byOrcid.set(orcidKey, normalizedAuthor); - } - - if (nameKey !== '|') { - byName.set(nameKey, normalizedAuthor); - } - } - - return deduped; -} - -function orderAuthorsByContributorRank(authors, contributorAuthors) { - const normalizedAuthors = normalizeAuthors(Array.isArray(authors) ? authors : []); - const normalizedContributors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); - - if (normalizedContributors.length === 0 || normalizedAuthors.length <= 1) { - return normalizedAuthors; - } - - const contributorOrcidIndex = new Map(); - const contributorNameKeyIndex = new Map(); - - normalizedContributors.forEach((contributorAuthor, index) => { - const orcidKey = cleanString(contributorAuthor?.orcid ?? '').toLowerCase(); - if (orcidKey && !contributorOrcidIndex.has(orcidKey)) { - contributorOrcidIndex.set(orcidKey, index); - } - - for (const key of authorAltNameKeys(contributorAuthor)) { - if (!contributorNameKeyIndex.has(key)) { - contributorNameKeyIndex.set(key, index); - } - } - }); - - const ranked = normalizedAuthors.map((author, originalIndex) => { - const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); - if (orcidKey && contributorOrcidIndex.has(orcidKey)) { - return { author, originalIndex, rank: contributorOrcidIndex.get(orcidKey) }; - } - - const nameRanks = authorAltNameKeys(author) - .map((key) => contributorNameKeyIndex.get(key)) - .filter((value) => Number.isInteger(value)); - - if (nameRanks.length > 0) { - return { author, originalIndex, rank: Math.min(...nameRanks) }; - } - - const heuristicIndex = normalizedContributors.findIndex((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); - if (heuristicIndex >= 0) { - return { author, originalIndex, rank: heuristicIndex }; - } - - return { author, originalIndex, rank: Number.POSITIVE_INFINITY }; - }); - - ranked.sort((left, right) => { - if (left.rank !== right.rank) { - return left.rank - right.rank; - } - - return left.originalIndex - right.originalIndex; - }); - - return ranked.map((entry) => entry.author); -} - -function isAutomatedContributor(contributor, profile) { - const login = cleanString(profile?.login ?? contributor?.login ?? '').toLowerCase(); - const contributorType = cleanString(contributor?.type ?? '').toLowerCase(); - const profileType = cleanString(profile?.type ?? '').toLowerCase(); - - if ((contributorType && contributorType !== 'user') || (profileType && profileType !== 'user')) { - return true; - } - - if (!login) { - return false; - } - - if (login.endsWith('[bot]')) { - return true; - } - - return /(^|[-_])(github-actions|dependabot|copilot|codex|claude|swe-agent)([-_]|$)/.test(login); -} diff --git a/src/services/githubImporterAuthors.js b/src/services/githubImporterAuthors.js new file mode 100644 index 0000000..65dee34 --- /dev/null +++ b/src/services/githubImporterAuthors.js @@ -0,0 +1,269 @@ +import { + cleanString, + normalizeAuthor, + normalizeAuthors, +} from './githubImporterUtils.js'; + +function normalizeNameToken(value) { + return cleanString(value) + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function authorMatchMetadata(author) { + const givenNames = normalizeNameToken(author?.givenNames ?? ''); + const familyNames = normalizeNameToken(author?.familyNames ?? ''); + const givenTokens = givenNames.split(' ').filter(Boolean); + const familyTokens = familyNames.split(' ').filter(Boolean); + + return { + givenNames, + familyNames, + givenFirst: givenTokens[0] ?? '', + givenInitials: givenTokens.map((token) => token[0]).join(''), + familyLast: familyTokens[familyTokens.length - 1] ?? '', + fullName: [givenNames, familyNames].filter(Boolean).join(' ').trim(), + }; +} + +function authorAltNameKeys(author) { + const metadata = authorMatchMetadata(author); + const givenNames = metadata.givenNames; + const familyNames = metadata.familyNames; + const givenFirst = metadata.givenFirst; + const familyLast = metadata.familyLast; + const keys = new Set([ + `${givenNames}|${familyNames}`, + `${givenFirst}|${familyNames}`, + `${givenNames}|${familyLast}`, + `${givenFirst}|${familyLast}`, + ]); + + keys.delete('|'); + keys.delete(''); + return [...keys].filter(Boolean); +} + +export function authorsLikelyMatch(sourceAuthor, contributorAuthor) { + const source = authorMatchMetadata(sourceAuthor); + const contributor = authorMatchMetadata(contributorAuthor); + + if (!source.familyLast || !contributor.familyLast || source.familyLast !== contributor.familyLast) { + return false; + } + + if (source.givenNames && contributor.givenNames && source.givenNames === contributor.givenNames) { + return true; + } + + if (source.givenFirst && contributor.givenFirst && source.givenFirst === contributor.givenFirst) { + return true; + } + + if (source.givenInitials && contributor.givenInitials && source.givenInitials === contributor.givenInitials) { + return true; + } + + if (source.fullName && contributor.fullName && source.fullName === contributor.fullName) { + return true; + } + + return false; +} + +export function enrichAuthorsWithContributorData(sourceAuthors, contributorAuthors) { + const normalizedSourceAuthors = normalizeAuthors(Array.isArray(sourceAuthors) ? sourceAuthors : []); + if (normalizedSourceAuthors.length === 0) { + return normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); + } + + const normalizedContributorAuthors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); + const contributorMatches = new Map(); + + for (const contributorAuthor of normalizedContributorAuthors) { + if (!contributorAuthor.orcid && !contributorAuthor.affiliation) { + continue; + } + + for (const key of authorAltNameKeys(contributorAuthor)) { + const existing = contributorMatches.get(key) ?? []; + existing.push(contributorAuthor); + contributorMatches.set(key, existing); + } + } + + return normalizedSourceAuthors.map((author) => { + const keyedMatches = authorAltNameKeys(author) + .flatMap((key) => contributorMatches.get(key) ?? []); + const heuristicMatches = normalizedContributorAuthors.filter((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); + const matches = [...new Set([...keyedMatches, ...heuristicMatches])]; + const uniqueOrcids = [...new Set(matches.map((match) => match.orcid).filter(Boolean))]; + const uniqueAffiliations = [...new Set(matches.map((match) => match.affiliation).filter(Boolean))]; + + if ((!author.orcid && uniqueOrcids.length > 1) || (!author.affiliation && uniqueAffiliations.length > 1)) { + return author; + } + + return { + ...author, + orcid: author.orcid || uniqueOrcids[0] || '', + affiliation: author.affiliation || uniqueAffiliations[0] || '', + }; + }); +} + +export function dedupeAuthors(authors) { + const seen = new Set(); + const byOrcid = new Map(); + const byName = new Map(); + const deduped = []; + + for (const rawAuthor of authors) { + const author = normalizeAuthor(rawAuthor); + if (!author) { + continue; + } + + const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); + const nameKey = [ + cleanString(author?.givenNames ?? '').toLowerCase(), + cleanString(author?.familyNames ?? '').toLowerCase(), + ].join('|'); + + if (nameKey !== '|' && byName.has(nameKey)) { + const existing = byName.get(nameKey); + const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); + const canMergeByName = !existingOrcid || !orcidKey || existingOrcid === orcidKey; + + if (canMergeByName) { + if (!existing.orcid && author.orcid) { + existing.orcid = author.orcid; + } + if (!existing.affiliation && author.affiliation) { + existing.affiliation = author.affiliation; + } + if (orcidKey && !byOrcid.has(orcidKey)) { + byOrcid.set(orcidKey, existing); + } + continue; + } + } + + const likelyMatch = deduped.find((existing) => { + const existingOrcid = cleanString(existing?.orcid ?? '').toLowerCase(); + const hasConflictingOrcid = existingOrcid && orcidKey && existingOrcid !== orcidKey; + + if (hasConflictingOrcid) { + return false; + } + + return authorsLikelyMatch(existing, author); + }); + + if (likelyMatch) { + if (!likelyMatch.orcid && author.orcid) { + likelyMatch.orcid = author.orcid; + } + if (!likelyMatch.affiliation && author.affiliation) { + likelyMatch.affiliation = author.affiliation; + } + + const mergedOrcidKey = cleanString(likelyMatch?.orcid ?? '').toLowerCase(); + if (mergedOrcidKey && !byOrcid.has(mergedOrcidKey)) { + byOrcid.set(mergedOrcidKey, likelyMatch); + } + continue; + } + + if (orcidKey && byOrcid.has(orcidKey)) { + const existing = byOrcid.get(orcidKey); + if (!existing.affiliation && author.affiliation) { + existing.affiliation = author.affiliation; + } + continue; + } + + const key = [ + cleanString(author?.givenNames ?? '').toLowerCase(), + cleanString(author?.familyNames ?? '').toLowerCase(), + cleanString(author?.orcid ?? '').toLowerCase(), + ].join('|'); + + if (!key || seen.has(key)) { + continue; + } + + seen.add(key); + const normalizedAuthor = { ...author }; + deduped.push(normalizedAuthor); + + if (orcidKey) { + byOrcid.set(orcidKey, normalizedAuthor); + } + + if (nameKey !== '|') { + byName.set(nameKey, normalizedAuthor); + } + } + + return deduped; +} + +export function orderAuthorsByContributorRank(authors, contributorAuthors) { + const normalizedAuthors = normalizeAuthors(Array.isArray(authors) ? authors : []); + const normalizedContributors = normalizeAuthors(Array.isArray(contributorAuthors) ? contributorAuthors : []); + + if (normalizedContributors.length === 0 || normalizedAuthors.length <= 1) { + return normalizedAuthors; + } + + const contributorOrcidIndex = new Map(); + const contributorNameKeyIndex = new Map(); + + normalizedContributors.forEach((contributorAuthor, index) => { + const orcidKey = cleanString(contributorAuthor?.orcid ?? '').toLowerCase(); + if (orcidKey && !contributorOrcidIndex.has(orcidKey)) { + contributorOrcidIndex.set(orcidKey, index); + } + + for (const key of authorAltNameKeys(contributorAuthor)) { + if (!contributorNameKeyIndex.has(key)) { + contributorNameKeyIndex.set(key, index); + } + } + }); + + const ranked = normalizedAuthors.map((author, originalIndex) => { + const orcidKey = cleanString(author?.orcid ?? '').toLowerCase(); + if (orcidKey && contributorOrcidIndex.has(orcidKey)) { + return { author, originalIndex, rank: contributorOrcidIndex.get(orcidKey) }; + } + + const nameRanks = authorAltNameKeys(author) + .map((key) => contributorNameKeyIndex.get(key)) + .filter((value) => Number.isInteger(value)); + + if (nameRanks.length > 0) { + return { author, originalIndex, rank: Math.min(...nameRanks) }; + } + + const heuristicIndex = normalizedContributors.findIndex((contributorAuthor) => authorsLikelyMatch(author, contributorAuthor)); + if (heuristicIndex >= 0) { + return { author, originalIndex, rank: heuristicIndex }; + } + + return { author, originalIndex, rank: Number.POSITIVE_INFINITY }; + }); + + ranked.sort((left, right) => { + if (left.rank !== right.rank) { + return left.rank - right.rank; + } + + return left.originalIndex - right.originalIndex; + }); + + return ranked.map((entry) => entry.author); +} diff --git a/src/services/githubImporterContributors.js b/src/services/githubImporterContributors.js new file mode 100644 index 0000000..f9f91fd --- /dev/null +++ b/src/services/githubImporterContributors.js @@ -0,0 +1,258 @@ +const API_BASE = 'https://api.github.com'; +const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; +const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; +const GITHUB_PAGE_SIZE = 100; + +async function fetchOrcidFromGithubProfileHtml(profileUrl, cleanString, extractOrcidFromGithubHtml) { + const url = cleanString(profileUrl); + if (!url) { + return null; + } + + try { + const response = await fetch(url, { + headers: { + Accept: 'text/html', + }, + }); + + if (!response.ok) { + return null; + } + + const html = await response.text(); + return extractOrcidFromGithubHtml(html); + } catch { + return null; + } +} + +function isAutomatedContributor(contributor, profile, cleanString) { + const login = cleanString(profile?.login ?? contributor?.login ?? '').toLowerCase(); + const contributorType = cleanString(contributor?.type ?? '').toLowerCase(); + const profileType = cleanString(profile?.type ?? '').toLowerCase(); + + if ((contributorType && contributorType !== 'user') || (profileType && profileType !== 'user')) { + return true; + } + + if (!login) { + return false; + } + + if (login.endsWith('[bot]')) { + return true; + } + + return /(^|[-_])(github-actions|dependabot|copilot|codex|claude|swe-agent)([-_]|$)/.test(login); +} + +async function fetchAllContributors(owner, repo, warnings, authToken, maxContributors, { fetchOptionalJson, addWarning }) { + const contributors = []; + let page = 1; + + while (true) { + const pageContributors = await fetchOptionalJson( + `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, + { + authToken, + source: 'contributors', + label: `contributors page ${page}`, + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }, + ) || []; + + if (!Array.isArray(pageContributors) || pageContributors.length === 0) { + break; + } + + contributors.push(...pageContributors); + + if (maxContributors && contributors.length >= maxContributors) { + return contributors.slice(0, maxContributors); + } + + if (pageContributors.length < GITHUB_PAGE_SIZE) { + break; + } + + page += 1; + } + + return contributors; +} + +export function resolveContributorFallbackLimit(options = {}) { + if (!Object.prototype.hasOwnProperty.call(options, 'contributorFallbackLimit')) { + return TOP_CONTRIBUTOR_FALLBACK_LIMIT; + } + + if (options.contributorFallbackLimit == null || options.contributorFallbackLimit === '') { + return null; + } + + const rawLimit = Number(options.contributorFallbackLimit); + + if (!Number.isFinite(rawLimit)) { + return TOP_CONTRIBUTOR_FALLBACK_LIMIT; + } + + return Math.min(Math.max(Math.trunc(rawLimit), 1), MAX_CONTRIBUTOR_FALLBACK_LIMIT); +} + +export async function fetchContributorAuthors({ + owner, + repo, + warnings, + authToken = '', + contributorFallbackLimit = TOP_CONTRIBUTOR_FALLBACK_LIMIT, + emitFallbackWarning = true, + cleanString, + normalizeAuthor, + normalizeAuthors, + dedupeAuthors, + addWarning, + fetchOptionalJson, + extractOrcidFromGithubProfile, + extractOrcidFromGithubHtml, +}) { + const contributors = await fetchAllContributors(owner, repo, warnings, authToken, contributorFallbackLimit, { + fetchOptionalJson, + addWarning, + }); + + if (!Array.isArray(contributors) || contributors.length === 0) { + return { + fallbackAuthors: [], + lookupAuthors: [], + }; + } + + if (emitFallbackWarning) { + addWarning( + warnings, + 'authors', + 'commit-based-fallback', + contributorFallbackLimit + ? `Using top ${contributorFallbackLimit} contributors as fallback authors.` + : 'Using contributors as fallback authors.', + { owner, repo }, + ); + } + + const profiles = await Promise.all( + contributors.map(async (contributor) => { + const login = cleanString(contributor?.login ?? ''); + if (!login) { + return { + contributor, + profile: null, + socialAccounts: [], + author: null, + autoFilledOrcid: false, + excludedAutomated: false, + }; + } + + const profile = await fetchOptionalJson( + `${API_BASE}/users/${encodeURIComponent(login)}`, + { + authToken, + source: 'contributor-profile', + label: `the profile for ${login}`, + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }, + ); + + const socialAccounts = await fetchOptionalJson( + `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, + { + authToken, + source: 'contributor-profile-links', + label: `the profile links for ${login}`, + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }, + ) || []; + + if (isAutomatedContributor(contributor, profile, cleanString)) { + return { + contributor, + profile, + socialAccounts, + author: null, + autoFilledOrcid: false, + excludedAutomated: true, + }; + } + + let profileOrcid = extractOrcidFromGithubProfile(profile, socialAccounts); + if (!profileOrcid) { + profileOrcid = await fetchOrcidFromGithubProfileHtml( + profile?.html_url ?? contributor?.html_url ?? '', + cleanString, + extractOrcidFromGithubHtml, + ); + } + + if (profile?.name) { + return { + contributor, + profile, + socialAccounts, + author: normalizeAuthor({ + name: profile.name, + affiliation: profile.company ?? '', + orcid: profileOrcid, + }), + autoFilledOrcid: Boolean(profileOrcid), + excludedAutomated: false, + }; + } + + return { + contributor, + profile, + socialAccounts, + author: normalizeAuthor({ + name: login, + affiliation: '', + orcid: profileOrcid, + }), + autoFilledOrcid: Boolean(profileOrcid), + excludedAutomated: false, + }; + }), + ); + + const excludedAutomatedCount = profiles.filter((entry) => entry?.excludedAutomated).length; + if (excludedAutomatedCount > 0) { + addWarning( + warnings, + 'authors', + 'automated-contributors-excluded', + `Excluded ${excludedAutomatedCount} automated account(s) from fallback authors.`, + { owner, repo }, + ); + } + + const autoFilledOrcidCount = profiles.filter((entry) => entry?.autoFilledOrcid).length; + if (autoFilledOrcidCount > 0) { + addWarning( + warnings, + 'authors', + 'orcid-autofilled', + `Auto-filled ORCID for ${autoFilledOrcidCount} contributor(s) from GitHub profile data.`, + { owner, repo }, + ); + } + + const fallbackAuthors = profiles + .slice(0, contributorFallbackLimit ?? profiles.length) + .map((entry) => entry?.author); + const lookupAuthors = profiles.map((entry) => entry?.author); + + return { + fallbackAuthors: dedupeAuthors(normalizeAuthors(fallbackAuthors)), + lookupAuthors: dedupeAuthors(normalizeAuthors(lookupAuthors)), + }; +} diff --git a/src/services/githubImporterParsers.js b/src/services/githubImporterParsers.js new file mode 100644 index 0000000..a985795 --- /dev/null +++ b/src/services/githubImporterParsers.js @@ -0,0 +1,170 @@ +import { + cleanString as utilCleanString, + extractFirstMarkdownParagraph as utilExtractFirstMarkdownParagraph, + normalizeAuthor as utilNormalizeAuthor, + normalizeKeywords as utilNormalizeKeywords, + normalizeRepoUrl as utilNormalizeRepoUrl, +} from './githubImporterUtils.js'; + +const cleanString = utilCleanString; +const normalizeAuthor = utilNormalizeAuthor; +const normalizeKeywords = utilNormalizeKeywords; +const normalizeRepoUrl = utilNormalizeRepoUrl; +const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; + +function parseJsonSafely(text) { + return JSON.parse(text); +} + +function extractPackageAuthors(payload) { + const candidates = []; + + if (payload.author) { + candidates.push(payload.author); + } + + if (Array.isArray(payload.authors)) { + candidates.push(...payload.authors); + } + + return candidates.map((item) => normalizeAuthor(item)).filter(Boolean); +} + +export function parsePackageJson(text) { + const payload = parseJsonSafely(text); + + return { + title: cleanString(payload.name ?? ''), + abstract: cleanString(payload.description ?? ''), + version: cleanString(payload.version ?? ''), + repositoryCode: normalizeRepoUrl(typeof payload.repository === 'string' ? payload.repository : payload.repository?.url ?? payload.homepage ?? ''), + license: cleanString(typeof payload.license === 'string' ? payload.license : payload.license?.type ?? ''), + keywords: normalizeKeywords(payload.keywords), + authors: extractPackageAuthors(payload), + }; +} + +function extractTomlSection(text, sectionName) { + const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); + const sectionLines = []; + let inSection = false; + + for (const line of lines) { + const sectionMatch = line.trim().match(/^\[([^\]]+)\]$/); + + if (sectionMatch) { + if (inSection) { + break; + } + + inSection = sectionMatch[1] === sectionName; + continue; + } + + if (inSection) { + sectionLines.push(line); + } + } + + return sectionLines.join('\n'); +} + +function extractTomlValue(sectionText, key) { + const match = sectionText.match(new RegExp(`^${key}\\s*=\\s*(.+)$`, 'm')); + return match ? match[1].trim() : ''; +} + +function parseTomlString(sectionText, key) { + const value = extractTomlValue(sectionText, key); + const match = value.match(/^['"](.+?)['"]$/); + return match ? match[1].trim() : ''; +} + +function parseTomlStrings(value) { + return [...String(value ?? '').matchAll(/['"]([^'"]+)['"]/g)].map((match) => cleanString(match[1])).filter(Boolean); +} + +export function parsePyprojectToml(text) { + const section = extractTomlSection(text, 'project') || extractTomlSection(text, 'tool.poetry'); + const authorsBlock = extractTomlValue(section, 'authors') || extractTomlValue(section, 'maintainers'); + const authors = [...String(authorsBlock ?? '').matchAll(/name\s*=\s*['"]([^'"]+)['"]/g)] + .map((match) => normalizeAuthor({ name: match[1] })) + .filter(Boolean); + + const licenseValue = parseTomlString(section, 'license') || cleanString((section.match(/license\s*=\s*\{[^}]*text\s*=\s*['"]([^'"]+)['"][^}]*\}/s) || [])[1] ?? ''); + const repositoryCode = parseTomlString(section, 'repository') || parseTomlString(section, 'homepage') || parseTomlString(section, 'url'); + + return { + title: parseTomlString(section, 'name'), + abstract: parseTomlString(section, 'description'), + version: parseTomlString(section, 'version'), + repositoryCode, + license: licenseValue, + keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), + authors, + }; +} + +export function parseSetupPy(text) { + const source = String(text ?? ''); + const extract = (key) => cleanString((source.match(new RegExp(`${key}\\s*=\\s*['"]([^'"]+)['"]`, 'm')) || [])[1] ?? ''); + + const authors = []; + const author = extract('author'); + const maintainer = extract('maintainer'); + + if (author) { + authors.push(normalizeAuthor({ name: author })); + } else if (maintainer) { + authors.push(normalizeAuthor({ name: maintainer })); + } + + return { + title: extract('name'), + abstract: extract('description'), + version: extract('version'), + repositoryCode: normalizeRepoUrl(extract('url')), + license: extract('license'), + keywords: normalizeKeywords(extract('keywords')), + authors: authors.filter(Boolean), + }; +} + +export function parseCargoToml(text) { + const section = extractTomlSection(text, 'package'); + const authors = parseTomlStrings(extractTomlValue(section, 'authors')).map((name) => normalizeAuthor({ name })).filter(Boolean); + + return { + title: parseTomlString(section, 'name'), + abstract: parseTomlString(section, 'description'), + version: parseTomlString(section, 'version'), + repositoryCode: parseTomlString(section, 'repository'), + license: parseTomlString(section, 'license'), + keywords: normalizeKeywords(parseTomlStrings(extractTomlValue(section, 'keywords'))), + authors, + }; +} + +export function parsePomXml(text) { + const source = String(text ?? ''); + const extract = (pattern) => cleanString((source.match(pattern) || [])[1] ?? ''); + const authors = [...source.matchAll(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/developer>/g)] + .map((match) => normalizeAuthor({ name: match[1] })) + .filter(Boolean); + + const licenseMatch = source.match(/[\s\S]*?([^<]+)<\/name>[\s\S]*?<\/license>/); + + return { + title: extract(/([^<]+)<\/name>/), + abstract: extract(/([^<]+)<\/description>/), + version: extract(/([^<]+)<\/version>/), + repositoryCode: extract(/([^<]+)<\/url>/), + license: cleanString((licenseMatch || [])[1] ?? ''), + keywords: [], + authors, + }; +} + +export function parseReadme(text) { + return extractFirstMarkdownParagraph(text); +} diff --git a/src/services/githubImporterUtils.js b/src/services/githubImporterUtils.js new file mode 100644 index 0000000..e45ac27 --- /dev/null +++ b/src/services/githubImporterUtils.js @@ -0,0 +1,271 @@ +import { normalizeOrcid } from '../utils/orcid.js'; + +function cleanString(value) { + return String(value ?? '').replace(/[\t ]+/g, ' ').trim(); +} + +function firstNonEmpty(...values) { + for (const value of values) { + if (Array.isArray(value)) { + if (value.length > 0) { + return value; + } + continue; + } + + const text = cleanString(value); + if (text) { + return text; + } + } + + return ''; +} + +function normalizeStringList(value) { + if (Array.isArray(value)) { + return value.map((item) => cleanString(item)).filter(Boolean); + } + + if (!value) { + return []; + } + + return String(value) + .split(/[\n,]/) + .map((item) => cleanString(item)) + .filter(Boolean); +} + +function normalizeKeywords(value) { + return [...new Set(normalizeStringList(value).map((keyword) => keyword.toLowerCase()))]; +} + +function normalizeReferences(value) { + if (Array.isArray(value)) { + return value.map((item) => cleanString(item)).filter(Boolean); + } + + if (!value) { + return []; + } + + return String(value) + .split(/\n+/) + .map((item) => cleanString(item)) + .filter(Boolean); +} + +function normalizeGrants(value) { + if (Array.isArray(value)) { + return value + .map((item) => { + if (typeof item === 'string') { + return cleanString(item); + } + + if (item && typeof item === 'object') { + return cleanString(item.id ?? item.value ?? item.grantId ?? ''); + } + + return ''; + }) + .filter(Boolean); + } + + if (!value) { + return []; + } + + return String(value) + .split(/\n+/) + .map((item) => cleanString(item)) + .filter(Boolean); +} + +function capitalizeToken(token) { + const text = cleanString(token); + if (!text) { + return ''; + } + + return text + .split(/([\-'])/) + .map((part) => { + if (part === '-' || part === "'") { + return part; + } + + // Preserve mixed-case tokens (for example, McDonald) and normalize others. + if (/[a-z]/.test(part) && /[A-Z]/.test(part)) { + return part; + } + + const lower = part.toLowerCase(); + return lower.charAt(0).toUpperCase() + lower.slice(1); + }) + .join(''); +} + +function capitalizeName(value) { + return cleanString(value) + .split(/\s+/) + .map((part) => capitalizeToken(part)) + .filter(Boolean) + .join(' '); +} + +function humanizeIdentifier(value) { + return cleanString(value) + .replace(/[._-]+/g, ' ') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .replace(/([a-z\d])([A-Z])/g, '$1 $2') + .replace(/([A-Za-z])(\d)/g, '$1 $2') + .replace(/(\d)([A-Za-z])/g, '$1 $2'); +} + +function splitDisplayName(name) { + const value = cleanString(name); + + if (!value) { + return { givenNames: '', familyNames: '' }; + } + + if (value.includes(',')) { + const [familyNames, ...givenParts] = value.split(','); + return { + givenNames: capitalizeName(givenParts.join(',').trim()), + familyNames: capitalizeName(familyNames), + }; + } + + const normalized = humanizeIdentifier(value); + const parts = normalized.split(/\s+/).filter(Boolean); + + if (parts.length <= 1) { + return { givenNames: capitalizeName(parts[0] ?? ''), familyNames: '' }; + } + + return { + givenNames: capitalizeName(parts.slice(0, -1).join(' ')), + familyNames: capitalizeName(parts[parts.length - 1]), + }; +} + +function normalizeAuthor(input) { + if (!input) { + return null; + } + + if (typeof input === 'string') { + const { givenNames, familyNames } = splitDisplayName(input); + return givenNames || familyNames ? { givenNames, familyNames, orcid: '', affiliation: '' } : null; + } + + if (typeof input !== 'object') { + return null; + } + + const name = cleanString(input.name ?? input.fullName ?? input.full_name ?? input.creator_name ?? ''); + const parsedName = name ? splitDisplayName(name) : null; + let givenNames = capitalizeName(input.givenNames ?? input['given-names'] ?? input.firstName ?? input.firstname ?? parsedName?.givenNames ?? ''); + let familyNames = capitalizeName(input.familyNames ?? input['family-names'] ?? input.lastName ?? input.lastname ?? parsedName?.familyNames ?? ''); + const affiliation = cleanString(input.affiliation ?? input.organization ?? input.company ?? input.institution ?? ''); + const orcid = normalizeOrcid(input.orcid ?? input.ORCID ?? input.orcidId ?? ''); + + if (givenNames && !familyNames) { + const reparsed = splitDisplayName(givenNames); + if (reparsed.familyNames) { + givenNames = reparsed.givenNames; + familyNames = reparsed.familyNames; + } + } + + if (!givenNames && !familyNames && !affiliation && !orcid) { + return null; + } + + return { givenNames, familyNames, orcid, affiliation }; +} + +function normalizeAuthors(value) { + if (!Array.isArray(value)) { + return []; + } + + return value.map((item) => normalizeAuthor(item)).filter(Boolean); +} + +function normalizeRepoUrl(value) { + const text = cleanString(value); + if (!text) { + return ''; + } + + const trimmed = text.replace(/^git\+/, '').replace(/\.git$/i, '').replace(/\/+$/, ''); + + try { + const parsed = new URL(trimmed); + const host = parsed.hostname.toLowerCase(); + let pathname = parsed.pathname.replace(/\/+$/, ''); + if (host === 'github.com') { + pathname = pathname.toLowerCase(); + } + return `${parsed.protocol}//${host}${pathname}`; + } catch { + return trimmed; + } +} + +function normalizeVersionForCompare(value) { + const text = cleanString(value).toLowerCase(); + if (!text) { + return ''; + } + + return text.replace(/^v(?=\d)/, ''); +} + +function extractFirstMarkdownParagraph(text) { + const lines = String(text ?? '').replace(/\r\n/g, '\n').split('\n'); + const paragraph = []; + let started = false; + + for (const line of lines) { + const trimmed = line.trim(); + + if (!trimmed) { + if (started) { + break; + } + continue; + } + + if (!started && /^#{1,6}\s+/.test(trimmed)) { + started = true; + continue; + } + + if (!started && /^(!|\[|-)/.test(trimmed)) { + continue; + } + + started = true; + paragraph.push(trimmed); + } + + return paragraph.join(' ').replace(/\s+/g, ' ').trim(); +} + +export { + cleanString, + extractFirstMarkdownParagraph, + firstNonEmpty, + normalizeAuthor, + normalizeAuthors, + normalizeGrants, + normalizeKeywords, + normalizeReferences, + normalizeRepoUrl, + normalizeVersionForCompare, +}; diff --git a/tests/services/citationValidation.test.js b/tests/services/citationValidation.test.js index b461cb5..a21316c 100644 --- a/tests/services/citationValidation.test.js +++ b/tests/services/citationValidation.test.js @@ -67,7 +67,15 @@ repository-code: "https://github.com/Imageomics/OpenCite" test('toCitationCff omits empty author name fields in references', () => { const output = toCitationCff({ title: 'OpenCite', - authors: [], + authors: [ + { + citationAuthor: { + 'given-names': 'Jane', + 'family-names': 'Doe', + orcid: '', + }, + }, + ], keywords: [], license: 'MIT', typeOfWork: 'software', From 13063b62e1eb9ecdd3ffe85c74ad2d643e1d0dc5 Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Wed, 12 Aug 2026 11:32:59 -0400 Subject: [PATCH 11/23] refactor: extract repeated importer helpers --- src/services/githubImporter.js | 43 ++++++++++++--------------- src/services/githubImporterUtils.js | 10 +++++++ tests/services/githubImporter.test.js | 8 +++++ 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 2961817..717ab69 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -21,6 +21,7 @@ import { normalizeReferences as utilNormalizeReferences, normalizeRepoUrl as utilNormalizeRepoUrl, normalizeVersionForCompare as utilNormalizeVersionForCompare, + stripWrappingQuotes as utilStripWrappingQuotes, } from './githubImporterUtils.js'; import { fetchContributorAuthors, @@ -64,6 +65,7 @@ const normalizeAuthor = utilNormalizeAuthor; const normalizeAuthors = utilNormalizeAuthors; const normalizeRepoUrl = utilNormalizeRepoUrl; const normalizeVersionForCompare = utilNormalizeVersionForCompare; +const stripWrappingQuotes = utilStripWrappingQuotes; const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; function makeIssue(kind, source, code, message, details = {}) { @@ -100,6 +102,19 @@ function shouldInspectRepositoryFiles(options = {}) { return options.inspectRepositoryFiles !== false; } +function addValidationWarnings(warnings, metaKind, path, validationResult) { + if (!validationResult || validationResult.isValid) { + return; + } + + const message = `${path} failed validation: ${validationResult.errors.join(' | ')}`; + addWarning(warnings, metaKind, `${metaKind}-file-invalid`, message, { path }); + + for (const warning of validationResult.warnings) { + addWarning(warnings, metaKind, `${metaKind}-file-warning`, `${path}: ${warning}`, { path }); + } +} + export function resolvePreferredCitationPath(fileContents = {}) { if (fileContents['CITATION.cff']) { return 'CITATION.cff'; @@ -140,18 +155,8 @@ export function summarizeImportedMetadataFiles(fileContents = {}) { if (!citationValidation.isValid) { summary.citation.valid = false; summary.citation.errors = [...citationValidation.errors]; - addWarning( - warnings, - 'citation', - 'citation-file-invalid', - `${preferredCitationPath} failed validation: ${citationValidation.errors.join(' | ')}`, - { path: preferredCitationPath }, - ); - } - - for (const warning of citationValidation.warnings) { - addWarning(warnings, 'citation', 'citation-file-warning', `${preferredCitationPath}: ${warning}`, { path: preferredCitationPath }); } + addValidationWarnings(warnings, 'citation', preferredCitationPath, citationValidation); } const zenodoPath = '.zenodo.json'; @@ -162,18 +167,8 @@ export function summarizeImportedMetadataFiles(fileContents = {}) { if (!zenodoValidation.isValid) { summary.zenodo.valid = false; summary.zenodo.errors = [...zenodoValidation.errors]; - addWarning( - warnings, - 'zenodo', - 'zenodo-file-invalid', - `${zenodoPath} failed validation: ${zenodoValidation.errors.join(' | ')}`, - { path: zenodoPath }, - ); - } - - for (const warning of zenodoValidation.warnings) { - addWarning(warnings, 'zenodo', 'zenodo-file-warning', `${zenodoPath}: ${warning}`, { path: zenodoPath }); } + addValidationWarnings(warnings, 'zenodo', zenodoPath, zenodoValidation); } return summary; @@ -246,7 +241,7 @@ export function parseCitationCff(text) { }; const assignScalar = (key, value) => { - const normalized = cleanString(value).replace(/^"|"$/g, ''); + const normalized = stripWrappingQuotes(value); if (!normalized) { return; @@ -327,7 +322,7 @@ export function parseCitationCff(text) { if (trimmed.startsWith('-')) { flushReference(); - const inline = cleanString(trimmed.slice(1)).replace(/^"|"$/g, ''); + const inline = stripWrappingQuotes(trimmed.slice(1)); if (!inline) { currentReference = {}; continue; diff --git a/src/services/githubImporterUtils.js b/src/services/githubImporterUtils.js index e45ac27..42a0c6e 100644 --- a/src/services/githubImporterUtils.js +++ b/src/services/githubImporterUtils.js @@ -4,6 +4,15 @@ function cleanString(value) { return String(value ?? '').replace(/[\t ]+/g, ' ').trim(); } +function stripWrappingQuotes(value) { + const text = cleanString(value); + if (!text) { + return ''; + } + + return text.replace(/^"|"$/g, '').replace(/^'|'$/g, ''); +} + function firstNonEmpty(...values) { for (const value of values) { if (Array.isArray(value)) { @@ -268,4 +277,5 @@ export { normalizeReferences, normalizeRepoUrl, normalizeVersionForCompare, + stripWrappingQuotes, }; diff --git a/tests/services/githubImporter.test.js b/tests/services/githubImporter.test.js index cac8f04..5984788 100644 --- a/tests/services/githubImporter.test.js +++ b/tests/services/githubImporter.test.js @@ -10,6 +10,7 @@ import { summarizeImportedMetadataFiles, validateImportedMetadataFiles, } from '../../src/services/githubImporter.js'; +import { stripWrappingQuotes } from '../../src/services/githubImporterUtils.js'; test('parseCitationCff extracts top-level fields from common CFF content', () => { const parsed = parseCitationCff(`cff-version: 1.2.0 @@ -35,6 +36,13 @@ authors: assert.equal(parsed.authors.length, 1); }); +test('stripWrappingQuotes removes matching quote wrappers without altering inner text', () => { + assert.equal(stripWrappingQuotes('"OpenCite"'), 'OpenCite'); + assert.equal(stripWrappingQuotes("'OpenCite'"), 'OpenCite'); + assert.equal(stripWrappingQuotes('OpenCite'), 'OpenCite'); + assert.equal(stripWrappingQuotes('"quoted \\"text\\""'), 'quoted \\"text\\"'); +}); + test('parseCitationCff emits warning for preferred-citation sections', () => { const parsed = parseCitationCff(`cff-version: 1.2.0 title: "OpenCite" From 43a3f233d9b94717f8ce3e3ad9b3783ce317f98c Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:11:08 -0400 Subject: [PATCH 12/23] refactor: integrate buildGithubRequestConfig into GitHub metadata importer and contributors --- src/services/githubApi.js | 8 ++++++-- src/services/githubImporter.js | 23 ++++++++++++++-------- src/services/githubImporterContributors.js | 14 +++++++------ tests/services/githubImporter.test.js | 18 +++++++++++++++++ 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/services/githubApi.js b/src/services/githubApi.js index 4b55d6c..bfce90c 100644 --- a/src/services/githubApi.js +++ b/src/services/githubApi.js @@ -87,6 +87,10 @@ export function resolveGithubToken(options = {}) { return ''; } +export function buildGithubRequestConfig({ authToken = '', source = '', label = '', onWarning = () => {} } = {}) { + return { authToken, source, label, onWarning }; +} + export function createGithubHeaders(token = '') { const headers = { Accept: 'application/vnd.github+json', @@ -174,12 +178,12 @@ export async function fetchLatestCommitDate(owner, repo, defaultBranch, { authTo const branchFilter = defaultBranch ? `&sha=${encodeURIComponent(defaultBranch)}` : ''; const commits = await fetchOptionalJson( `${API_BASE}/repos/${owner}/${repo}/commits?per_page=1${branchFilter}`, - { + buildGithubRequestConfig({ authToken, source: 'commits', label: 'the latest commit', onWarning, - }, + }), ); if (!Array.isArray(commits) || commits.length === 0) { diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 717ab69..04e8cbe 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -3,6 +3,7 @@ import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile } from '../ut import { validateCitationCffText } from './citationValidation.js'; import { runCitationHealthScan } from './citationHealthScan.js'; import { + buildGithubRequestConfig, fetchContentsFile, fetchLatestCommitDate, fetchOptionalJson, @@ -704,12 +705,15 @@ export async function importGithubMetadata(repoUrl, options = {}) { } const defaultBranch = cleanString(repoData.default_branch ?? ''); - const releaseData = await fetchOptionalJson(`${API_BASE}/repos/${owner}/${repo}/releases/latest`, { - authToken, - source: 'release', - label: 'the latest release', - onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }); + const releaseData = await fetchOptionalJson( + `${API_BASE}/repos/${owner}/${repo}/releases/latest`, + buildGithubRequestConfig({ + authToken, + source: 'release', + label: 'the latest release', + onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), + }), + ); const latestCommitDate = releaseData?.published_at ? '' : await fetchLatestCommitDate(owner, repo, defaultBranch, { @@ -724,12 +728,15 @@ export async function importGithubMetadata(repoUrl, options = {}) { if (inspectRepositoryFiles) { const branchInfo = defaultBranch - ? await fetchOptionalJson(`${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(defaultBranch)}`, { + ? await fetchOptionalJson( + `${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(defaultBranch)}`, + buildGithubRequestConfig({ authToken, source: 'branch', label: 'the default branch', onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }) + }), + ) : null; ref = cleanString(branchInfo?.name ?? defaultBranch ?? repoData.default_branch ?? 'HEAD'); diff --git a/src/services/githubImporterContributors.js b/src/services/githubImporterContributors.js index f9f91fd..83a7d95 100644 --- a/src/services/githubImporterContributors.js +++ b/src/services/githubImporterContributors.js @@ -1,3 +1,5 @@ +import { buildGithubRequestConfig } from './githubApi.js'; + const API_BASE = 'https://api.github.com'; const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; @@ -54,12 +56,12 @@ async function fetchAllContributors(owner, repo, warnings, authToken, maxContrib while (true) { const pageContributors = await fetchOptionalJson( `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, - { + buildGithubRequestConfig({ authToken, source: 'contributors', label: `contributors page ${page}`, onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, + }), ) || []; if (!Array.isArray(pageContributors) || pageContributors.length === 0) { @@ -156,22 +158,22 @@ export async function fetchContributorAuthors({ const profile = await fetchOptionalJson( `${API_BASE}/users/${encodeURIComponent(login)}`, - { + buildGithubRequestConfig({ authToken, source: 'contributor-profile', label: `the profile for ${login}`, onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, + }), ); const socialAccounts = await fetchOptionalJson( `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, - { + buildGithubRequestConfig({ authToken, source: 'contributor-profile-links', label: `the profile links for ${login}`, onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details), - }, + }), ) || []; if (isAutomatedContributor(contributor, profile, cleanString)) { diff --git a/tests/services/githubImporter.test.js b/tests/services/githubImporter.test.js index 5984788..1a63d0f 100644 --- a/tests/services/githubImporter.test.js +++ b/tests/services/githubImporter.test.js @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { buildGithubRequestConfig } from '../../src/services/githubApi.js'; import { addCitationConsistencyWarnings, importGithubMetadata, @@ -36,6 +37,23 @@ authors: assert.equal(parsed.authors.length, 1); }); +test('buildGithubRequestConfig returns the same GitHub request-field shape', () => { + const onWarning = () => {}; + const config = buildGithubRequestConfig({ + authToken: 'token-123', + source: 'release', + label: 'the latest release', + onWarning, + }); + + assert.deepEqual(config, { + authToken: 'token-123', + source: 'release', + label: 'the latest release', + onWarning, + }); +}); + test('stripWrappingQuotes removes matching quote wrappers without altering inner text', () => { assert.equal(stripWrappingQuotes('"OpenCite"'), 'OpenCite'); assert.equal(stripWrappingQuotes("'OpenCite'"), 'OpenCite'); From 9b4e7cf8a00eb044d5cdfb32c7b25c4528b3bd7b Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:25:35 -0400 Subject: [PATCH 13/23] refactor: modularize GitHub API calls and improve metadata handling in importer --- src/services/github.js | 11 +- src/services/githubApi.js | 33 ++++ src/services/githubImporter.js | 187 +-------------------- src/services/githubImporterContributors.js | 16 +- src/services/githubImporterMerge.js | 159 ++++++++++++++++++ 5 files changed, 217 insertions(+), 189 deletions(-) create mode 100644 src/services/githubImporterMerge.js diff --git a/src/services/github.js b/src/services/github.js index c6053ac..f509ce5 100644 --- a/src/services/github.js +++ b/src/services/github.js @@ -1,4 +1,9 @@ import { createMetadata } from '../core/metadataModel.js'; +import { + buildGithubCommitListApiUrl, + buildGithubReleaseApiUrl, + buildGithubRepoApiUrl, +} from './githubApi.js'; /** * Parse a GitHub repository URL to extract owner and repo name @@ -37,7 +42,7 @@ function parseGithubUrl(url) { * @throws {Error} if API request fails */ async function fetchRepoData(owner, repo) { - const url = `https://api.github.com/repos/${owner}/${repo}`; + const url = buildGithubRepoApiUrl(owner, repo); const response = await fetch(url, { headers: { Accept: 'application/vnd.github.v3+json', @@ -61,7 +66,7 @@ async function fetchRepoData(owner, repo) { * @returns {Promise} Latest release object or null if no releases */ async function fetchLatestRelease(owner, repo) { - const url = `https://api.github.com/repos/${owner}/${repo}/releases/latest`; + const url = buildGithubReleaseApiUrl(owner, repo); const response = await fetch(url, { headers: { Accept: 'application/vnd.github.v3+json', @@ -95,7 +100,7 @@ async function fetchDefaultBranchSha(owner, repo, defaultBranch) { return null; } - const url = `https://api.github.com/repos/${owner}/${repo}/commits?sha=${encodeURIComponent(defaultBranch)}&per_page=1`; + const url = buildGithubCommitListApiUrl(owner, repo, defaultBranch); const response = await fetch(url, { headers: { Accept: 'application/vnd.github.v3+json', diff --git a/src/services/githubApi.js b/src/services/githubApi.js index bfce90c..e4e6fb4 100644 --- a/src/services/githubApi.js +++ b/src/services/githubApi.js @@ -91,6 +91,39 @@ export function buildGithubRequestConfig({ authToken = '', source = '', label = return { authToken, source, label, onWarning }; } +export function buildGithubRepoApiUrl(owner, repo) { + return `${API_BASE}/repos/${owner}/${repo}`; +} + +export function buildGithubReleaseApiUrl(owner, repo) { + return `${API_BASE}/repos/${owner}/${repo}/releases/latest`; +} + +export function buildGithubCommitListApiUrl(owner, repo, defaultBranch = '') { + const branchFilter = defaultBranch ? `&sha=${encodeURIComponent(defaultBranch)}` : ''; + return `${API_BASE}/repos/${owner}/${repo}/commits?per_page=1${branchFilter}`; +} + +export function buildGithubBranchApiUrl(owner, repo, branch) { + return `${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`; +} + +export function buildGithubContentsApiUrl(owner, repo, path, ref) { + return `${API_BASE}/repos/${owner}/${repo}/contents/${encodePath(path)}?ref=${encodeURIComponent(ref)}`; +} + +export function buildGithubContributorsApiUrl(owner, repo, page, perPage = 100) { + return `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${perPage}&page=${page}`; +} + +export function buildGithubUserApiUrl(login) { + return `${API_BASE}/users/${encodeURIComponent(login)}`; +} + +export function buildGithubUserSocialAccountsApiUrl(login) { + return `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`; +} + export function createGithubHeaders(token = '') { const headers = { Accept: 'application/vnd.github+json', diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 04e8cbe..6789b57 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -3,6 +3,8 @@ import { extractOrcidFromGithubHtml, extractOrcidFromGithubProfile } from '../ut import { validateCitationCffText } from './citationValidation.js'; import { runCitationHealthScan } from './citationHealthScan.js'; import { + buildGithubBranchApiUrl, + buildGithubReleaseApiUrl, buildGithubRequestConfig, fetchContentsFile, fetchLatestCommitDate, @@ -28,11 +30,8 @@ import { fetchContributorAuthors, resolveContributorFallbackLimit, } from './githubImporterContributors.js'; -import { - dedupeAuthors, - enrichAuthorsWithContributorData, - orderAuthorsByContributorRank, -} from './githubImporterAuthors.js'; +import { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; +export { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; import { parseCargoToml, parsePackageJson, @@ -45,7 +44,6 @@ import { compareExistingMetadataFiles } from './metadataComparison.js'; import { runMetadataReviewPipeline } from './metadataReview.js'; import { validateZenodoJsonText } from './zenodoValidation.js'; -const API_BASE = 'https://api.github.com'; const FILES_TO_INSPECT = [ 'CITATION.cff', '.zenodo.json', @@ -61,11 +59,9 @@ const cleanString = utilCleanString; const firstNonEmpty = utilFirstNonEmpty; const normalizeKeywords = utilNormalizeKeywords; const normalizeReferences = utilNormalizeReferences; -const normalizeGrants = utilNormalizeGrants; const normalizeAuthor = utilNormalizeAuthor; const normalizeAuthors = utilNormalizeAuthors; const normalizeRepoUrl = utilNormalizeRepoUrl; -const normalizeVersionForCompare = utilNormalizeVersionForCompare; const stripWrappingQuotes = utilStripWrappingQuotes; const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; @@ -509,174 +505,6 @@ function parseFile(path, text, warnings, errors) { return null; } -function mapTypeOfWork(value) { - const type = cleanString(value).toLowerCase(); - - if (type === 'dataset') { - return 'dataset'; - } - - if (['article', 'book', 'book-chapter', 'conference-paper', 'journal-article', 'manuscript', 'preprint', 'report', 'thesis'].includes(type)) { - return 'article'; - } - - if (type === 'other') { - return 'other'; - } - - return 'software'; -} - -function mergeMetadata({ - repo, - release, - defaultPublicationDate, - citation, - zenodo, - packageMeta, - readme, - contributors, - contributorLookupAuthors, - supplementalCitationAuthors = [], -}) { - const primaryAuthors = [ - ...normalizeAuthors(Array.isArray(citation?.authors) ? citation.authors : []), - ...normalizeAuthors(Array.isArray(zenodo?.authors) ? zenodo.authors : []), - ...normalizeAuthors(Array.isArray(packageMeta?.authors) ? packageMeta.authors : []), - ...normalizeAuthors(Array.isArray(supplementalCitationAuthors) ? supplementalCitationAuthors : []), - ]; - const authors = [ - ...normalizeAuthors(primaryAuthors), - ...normalizeAuthors(Array.isArray(contributors) ? contributors : []), - ]; - const keywords = normalizeKeywords(firstNonEmpty(citation?.keywords, zenodo?.keywords, packageMeta?.keywords, repo?.topics)); - const references = normalizeReferences(firstNonEmpty(zenodo?.references, citation?.references)); - const grants = normalizeGrants(firstNonEmpty(zenodo?.grants)); - const enrichedAuthors = enrichAuthorsWithContributorData(authors, contributorLookupAuthors); - const dedupedAuthors = dedupeAuthors(enrichedAuthors); - const orderedAuthors = orderAuthorsByContributorRank(dedupedAuthors, contributorLookupAuthors); - - return createMetadata({ - title: cleanString(firstNonEmpty(citation?.title, zenodo?.title, packageMeta?.title, repo?.name)), - authors: orderedAuthors, - keywords, - license: cleanString(firstNonEmpty(citation?.license, zenodo?.license, packageMeta?.license, repo?.license?.spdx_id)), - typeOfWork: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), - customTypeOfWork: '', - zenodoUploadType: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), - // Prefer metadata file versions for pre-release authoring; fall back to latest release tag. - version: cleanString(firstNonEmpty(citation?.version, zenodo?.version, packageMeta?.version, release?.tag_name)), - publicationDate: cleanString(firstNonEmpty(release?.published_at, citation?.publicationDate, zenodo?.publicationDate, defaultPublicationDate)).split('T')[0], - repositoryCode: normalizeRepoUrl(firstNonEmpty(repo?.html_url, citation?.repositoryCode, packageMeta?.repositoryCode)), - doi: cleanString(firstNonEmpty(zenodo?.doi, citation?.doi)), - abstract: cleanString(firstNonEmpty(citation?.abstract, zenodo?.abstract, packageMeta?.abstract, readme, repo?.description)), - references, - grants, - }); -} - -export function addCitationConsistencyWarnings({ warnings, citation, zenodo, releaseData, repoData, metadata }) { - const releaseTag = cleanString(releaseData?.tag_name ?? ''); - const citationVersion = cleanString(citation?.version ?? ''); - const zenodoVersion = cleanString(zenodo?.version ?? ''); - const finalVersion = cleanString(metadata?.version ?? ''); - const normalizedReleaseTag = normalizeVersionForCompare(releaseTag); - const normalizedCitationVersion = normalizeVersionForCompare(citationVersion); - const normalizedZenodoVersion = normalizeVersionForCompare(zenodoVersion); - - if (normalizedReleaseTag && normalizedCitationVersion && normalizedReleaseTag !== normalizedCitationVersion) { - addWarning( - warnings, - 'citation', - 'version-mismatch', - `CITATION.cff version (${citationVersion}) differs from latest release tag (${releaseTag}); using CITATION.cff version for import.`, - ); - } - - if (normalizedReleaseTag && normalizedZenodoVersion && normalizedReleaseTag !== normalizedZenodoVersion) { - addWarning( - warnings, - 'zenodo', - 'version-mismatch', - `.zenodo.json version (${zenodoVersion}) differs from latest release tag (${releaseTag}); using .zenodo.json version for import.`, - ); - } - - if (normalizedCitationVersion && normalizedZenodoVersion && normalizedCitationVersion !== normalizedZenodoVersion) { - addWarning( - warnings, - 'citation', - 'cross-file-version-mismatch', - `CITATION.cff version (${citationVersion}) and .zenodo.json version (${zenodoVersion}) differ.`, - ); - } - - const citationDate = cleanString(citation?.publicationDate ?? '').split('T')[0]; - const zenodoDate = cleanString(zenodo?.publicationDate ?? '').split('T')[0]; - const releaseDate = cleanString(releaseData?.published_at ?? '').split('T')[0]; - - if (releaseDate && citationDate && releaseDate !== citationDate) { - addWarning( - warnings, - 'citation', - 'date-mismatch', - `CITATION.cff date-released (${citationDate}) differs from latest release date (${releaseDate}); using release date for import.`, - ); - } - - if (releaseDate && zenodoDate && releaseDate !== zenodoDate) { - addWarning( - warnings, - 'zenodo', - 'date-mismatch', - `.zenodo.json publication_date (${zenodoDate}) differs from latest release date (${releaseDate}); using release date for import.`, - ); - } - - const repoUrl = normalizeRepoUrl(repoData?.html_url ?? ''); - const citationRepoUrl = normalizeRepoUrl(citation?.repositoryCode ?? ''); - - if (repoUrl && citationRepoUrl && repoUrl !== citationRepoUrl) { - addWarning( - warnings, - 'citation', - 'repository-url-mismatch', - `CITATION.cff repository-code (${citationRepoUrl}) differs from repository URL (${repoUrl}); using repository URL for import.`, - ); - } - - if (!finalVersion) { - addWarning( - warnings, - 'citation', - 'missing-version', - 'No version could be determined from release tag, CITATION.cff, .zenodo.json, or package metadata.', - ); - } - - const repoSpdx = cleanString(repoData?.license?.spdx_id ?? '').toUpperCase(); - const citationLicense = cleanString(citation?.license ?? '').toUpperCase(); - const zenodoLicense = cleanString(zenodo?.license ?? '').toUpperCase(); - - if (repoSpdx && citationLicense && repoSpdx !== citationLicense) { - addWarning( - warnings, - 'citation', - 'license-mismatch', - `CITATION.cff license (${citationLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`, - ); - } - - if (repoSpdx && zenodoLicense && repoSpdx !== zenodoLicense) { - addWarning( - warnings, - 'zenodo', - 'license-mismatch', - `.zenodo.json license (${zenodoLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`, - ); - } -} - export async function importGithubMetadata(repoUrl, options = {}) { const warnings = []; const errors = []; @@ -695,7 +523,7 @@ export async function importGithubMetadata(repoUrl, options = {}) { return { metadata: emptyMetadata, warnings, errors, review: null, healthScan: [] }; } - const repoData = await fetchRequiredJson(`${API_BASE}/repos/${owner}/${repo}`, { + const repoData = await fetchRequiredJson(`https://api.github.com/repos/${owner}/${repo}`, { authToken, source: 'repository', onError: (source, code, message, details = {}) => addError(errors, source, code, message, details), @@ -706,7 +534,7 @@ export async function importGithubMetadata(repoUrl, options = {}) { const defaultBranch = cleanString(repoData.default_branch ?? ''); const releaseData = await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/releases/latest`, + buildGithubReleaseApiUrl(owner, repo), buildGithubRequestConfig({ authToken, source: 'release', @@ -729,7 +557,7 @@ export async function importGithubMetadata(repoUrl, options = {}) { if (inspectRepositoryFiles) { const branchInfo = defaultBranch ? await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/branches/${encodeURIComponent(defaultBranch)}`, + buildGithubBranchApiUrl(owner, repo, defaultBranch), buildGithubRequestConfig({ authToken, source: 'branch', @@ -827,7 +655,6 @@ export async function importGithubMetadata(repoUrl, options = {}) { cleanString, normalizeAuthor, normalizeAuthors, - dedupeAuthors, addWarning, fetchOptionalJson, extractOrcidFromGithubProfile, diff --git a/src/services/githubImporterContributors.js b/src/services/githubImporterContributors.js index 83a7d95..6000a3b 100644 --- a/src/services/githubImporterContributors.js +++ b/src/services/githubImporterContributors.js @@ -1,6 +1,11 @@ -import { buildGithubRequestConfig } from './githubApi.js'; +import { + buildGithubContributorsApiUrl, + buildGithubRequestConfig, + buildGithubUserApiUrl, + buildGithubUserSocialAccountsApiUrl, +} from './githubApi.js'; +import { dedupeAuthors } from './githubImporterAuthors.js'; -const API_BASE = 'https://api.github.com'; const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4; const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20; const GITHUB_PAGE_SIZE = 100; @@ -55,7 +60,7 @@ async function fetchAllContributors(owner, repo, warnings, authToken, maxContrib while (true) { const pageContributors = await fetchOptionalJson( - `${API_BASE}/repos/${owner}/${repo}/contributors?per_page=${GITHUB_PAGE_SIZE}&page=${page}`, + buildGithubContributorsApiUrl(owner, repo, page, GITHUB_PAGE_SIZE), buildGithubRequestConfig({ authToken, source: 'contributors', @@ -112,7 +117,6 @@ export async function fetchContributorAuthors({ cleanString, normalizeAuthor, normalizeAuthors, - dedupeAuthors, addWarning, fetchOptionalJson, extractOrcidFromGithubProfile, @@ -157,7 +161,7 @@ export async function fetchContributorAuthors({ } const profile = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}`, + buildGithubUserApiUrl(login), buildGithubRequestConfig({ authToken, source: 'contributor-profile', @@ -167,7 +171,7 @@ export async function fetchContributorAuthors({ ); const socialAccounts = await fetchOptionalJson( - `${API_BASE}/users/${encodeURIComponent(login)}/social_accounts`, + buildGithubUserSocialAccountsApiUrl(login), buildGithubRequestConfig({ authToken, source: 'contributor-profile-links', diff --git a/src/services/githubImporterMerge.js b/src/services/githubImporterMerge.js new file mode 100644 index 0000000..3962de2 --- /dev/null +++ b/src/services/githubImporterMerge.js @@ -0,0 +1,159 @@ +import { createMetadata } from '../core/metadataModel.js'; +import { + cleanString, + normalizeAuthors, + normalizeGrants, + normalizeKeywords, + normalizeReferences, + normalizeRepoUrl, + normalizeVersionForCompare, +} from './githubImporterUtils.js'; +import { + dedupeAuthors, + enrichAuthorsWithContributorData, + orderAuthorsByContributorRank, +} from './githubImporterAuthors.js'; + +function firstNonEmpty(...values) { + for (const value of values) { + if (Array.isArray(value)) { + if (value.length > 0) { + return value; + } + continue; + } + + const text = cleanString(value); + if (text) { + return text; + } + } + + return ''; +} + +function addWarning(warnings, source, code, message) { + warnings.push({ kind: 'warning', source, code, message }); +} + +function mapTypeOfWork(value) { + const type = cleanString(value).toLowerCase(); + + if (type === 'dataset') { + return 'dataset'; + } + + if (['article', 'book', 'book-chapter', 'conference-paper', 'journal-article', 'manuscript', 'preprint', 'report', 'thesis'].includes(type)) { + return 'article'; + } + + if (type === 'other') { + return 'other'; + } + + return 'software'; +} + +export function mergeMetadata({ + repo, + release, + defaultPublicationDate, + citation, + zenodo, + packageMeta, + readme, + contributors, + contributorLookupAuthors, + supplementalCitationAuthors = [], +}) { + const primaryAuthors = [ + ...normalizeAuthors(Array.isArray(citation?.authors) ? citation.authors : []), + ...normalizeAuthors(Array.isArray(zenodo?.authors) ? zenodo.authors : []), + ...normalizeAuthors(Array.isArray(packageMeta?.authors) ? packageMeta.authors : []), + ...normalizeAuthors(Array.isArray(supplementalCitationAuthors) ? supplementalCitationAuthors : []), + ]; + const authors = [ + ...normalizeAuthors(primaryAuthors), + ...normalizeAuthors(Array.isArray(contributors) ? contributors : []), + ]; + const keywords = normalizeKeywords(firstNonEmpty(citation?.keywords, zenodo?.keywords, packageMeta?.keywords, repo?.topics)); + const references = normalizeReferences(firstNonEmpty(zenodo?.references, citation?.references)); + const grants = normalizeGrants(firstNonEmpty(zenodo?.grants)); + const enrichedAuthors = enrichAuthorsWithContributorData(authors, contributorLookupAuthors); + const dedupedAuthors = dedupeAuthors(enrichedAuthors); + const orderedAuthors = orderAuthorsByContributorRank(dedupedAuthors, contributorLookupAuthors); + + return createMetadata({ + title: cleanString(firstNonEmpty(citation?.title, zenodo?.title, packageMeta?.title, repo?.name)), + authors: orderedAuthors, + keywords, + license: cleanString(firstNonEmpty(citation?.license, zenodo?.license, packageMeta?.license, repo?.license?.spdx_id)), + typeOfWork: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), + customTypeOfWork: '', + zenodoUploadType: mapTypeOfWork(firstNonEmpty(zenodo?.typeOfWork, citation?.typeOfWork, 'software')), + version: cleanString(firstNonEmpty(citation?.version, zenodo?.version, packageMeta?.version, release?.tag_name)), + publicationDate: cleanString(firstNonEmpty(release?.published_at, citation?.publicationDate, zenodo?.publicationDate, defaultPublicationDate)).split('T')[0], + repositoryCode: normalizeRepoUrl(firstNonEmpty(repo?.html_url, citation?.repositoryCode, packageMeta?.repositoryCode)), + doi: cleanString(firstNonEmpty(zenodo?.doi, citation?.doi)), + abstract: cleanString(firstNonEmpty(citation?.abstract, zenodo?.abstract, packageMeta?.abstract, readme, repo?.description)), + references, + grants, + }); +} + +export function addCitationConsistencyWarnings({ warnings, citation, zenodo, releaseData, repoData, metadata }) { + const releaseTag = cleanString(releaseData?.tag_name ?? ''); + const citationVersion = cleanString(citation?.version ?? ''); + const zenodoVersion = cleanString(zenodo?.version ?? ''); + const finalVersion = cleanString(metadata?.version ?? ''); + const normalizedReleaseTag = normalizeVersionForCompare(releaseTag); + const normalizedCitationVersion = normalizeVersionForCompare(citationVersion); + const normalizedZenodoVersion = normalizeVersionForCompare(zenodoVersion); + + if (normalizedReleaseTag && normalizedCitationVersion && normalizedReleaseTag !== normalizedCitationVersion) { + addWarning(warnings, 'citation', 'version-mismatch', `CITATION.cff version (${citationVersion}) differs from latest release tag (${releaseTag}); using CITATION.cff version for import.`); + } + + if (normalizedReleaseTag && normalizedZenodoVersion && normalizedReleaseTag !== normalizedZenodoVersion) { + addWarning(warnings, 'zenodo', 'version-mismatch', `.zenodo.json version (${zenodoVersion}) differs from latest release tag (${releaseTag}); using .zenodo.json version for import.`); + } + + if (normalizedCitationVersion && normalizedZenodoVersion && normalizedCitationVersion !== normalizedZenodoVersion) { + addWarning(warnings, 'citation', 'cross-file-version-mismatch', `CITATION.cff version (${citationVersion}) and .zenodo.json version (${zenodoVersion}) differ.`); + } + + const citationDate = cleanString(citation?.publicationDate ?? '').split('T')[0]; + const zenodoDate = cleanString(zenodo?.publicationDate ?? '').split('T')[0]; + const releaseDate = cleanString(releaseData?.published_at ?? '').split('T')[0]; + + if (releaseDate && citationDate && releaseDate !== citationDate) { + addWarning(warnings, 'citation', 'date-mismatch', `CITATION.cff date-released (${citationDate}) differs from latest release date (${releaseDate}); using release date for import.`); + } + + if (releaseDate && zenodoDate && releaseDate !== zenodoDate) { + addWarning(warnings, 'zenodo', 'date-mismatch', `.zenodo.json publication_date (${zenodoDate}) differs from latest release date (${releaseDate}); using release date for import.`); + } + + const repoUrl = normalizeRepoUrl(repoData?.html_url ?? ''); + const citationRepoUrl = normalizeRepoUrl(citation?.repositoryCode ?? ''); + + if (repoUrl && citationRepoUrl && repoUrl !== citationRepoUrl) { + addWarning(warnings, 'citation', 'repository-url-mismatch', `CITATION.cff repository-code (${citationRepoUrl}) differs from repository URL (${repoUrl}); using repository URL for import.`); + } + + if (!finalVersion) { + addWarning(warnings, 'citation', 'missing-version', 'No version could be determined from release tag, CITATION.cff, .zenodo.json, or package metadata.'); + } + + const repoSpdx = cleanString(repoData?.license?.spdx_id ?? '').toUpperCase(); + const citationLicense = cleanString(citation?.license ?? '').toUpperCase(); + const zenodoLicense = cleanString(zenodo?.license ?? '').toUpperCase(); + + if (repoSpdx && citationLicense && repoSpdx !== citationLicense) { + addWarning(warnings, 'citation', 'license-mismatch', `CITATION.cff license (${citationLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`); + } + + if (repoSpdx && zenodoLicense && repoSpdx !== zenodoLicense) { + addWarning(warnings, 'zenodo', 'license-mismatch', `.zenodo.json license (${zenodoLicense}) differs from repository SPDX license (${repoSpdx}); imported metadata keeps source precedence but should be reviewed.`); + } +} From 5fbb5a25768589b85ede0102a0e1d6151fbbc62d Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:38:38 -0400 Subject: [PATCH 14/23] refactor: enhance stripWrappingQuotes function to handle mismatched quotes --- src/services/githubImporterUtils.js | 9 ++++++++- tests/services/githubImporter.test.js | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/services/githubImporterUtils.js b/src/services/githubImporterUtils.js index 42a0c6e..947cbd7 100644 --- a/src/services/githubImporterUtils.js +++ b/src/services/githubImporterUtils.js @@ -10,7 +10,14 @@ function stripWrappingQuotes(value) { return ''; } - return text.replace(/^"|"$/g, '').replace(/^'|'$/g, ''); + const first = text[0]; + const last = text[text.length - 1]; + + if ((first === '"' && last === '"') || (first === '\'' && last === '\'')) { + return text.slice(1, -1); + } + + return text; } function firstNonEmpty(...values) { diff --git a/tests/services/githubImporter.test.js b/tests/services/githubImporter.test.js index 1a63d0f..6397234 100644 --- a/tests/services/githubImporter.test.js +++ b/tests/services/githubImporter.test.js @@ -59,6 +59,7 @@ test('stripWrappingQuotes removes matching quote wrappers without altering inner assert.equal(stripWrappingQuotes("'OpenCite'"), 'OpenCite'); assert.equal(stripWrappingQuotes('OpenCite'), 'OpenCite'); assert.equal(stripWrappingQuotes('"quoted \\"text\\""'), 'quoted \\"text\\"'); + assert.equal(stripWrappingQuotes('"OpenCite\''), '"OpenCite\''); }); test('parseCitationCff emits warning for preferred-citation sections', () => { From 6a5fb7cd06e4711bf8b577cc8465e083b80997ba Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:40:42 -0400 Subject: [PATCH 15/23] refactor: remove redundant firstNonEmpty function and import it from core utilities --- src/services/githubImporterMerge.js | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/services/githubImporterMerge.js b/src/services/githubImporterMerge.js index 3962de2..d32f156 100644 --- a/src/services/githubImporterMerge.js +++ b/src/services/githubImporterMerge.js @@ -1,6 +1,7 @@ import { createMetadata } from '../core/metadataModel.js'; import { cleanString, + firstNonEmpty, normalizeAuthors, normalizeGrants, normalizeKeywords, @@ -14,24 +15,6 @@ import { orderAuthorsByContributorRank, } from './githubImporterAuthors.js'; -function firstNonEmpty(...values) { - for (const value of values) { - if (Array.isArray(value)) { - if (value.length > 0) { - return value; - } - continue; - } - - const text = cleanString(value); - if (text) { - return text; - } - } - - return ''; -} - function addWarning(warnings, source, code, message) { warnings.push({ kind: 'warning', source, code, message }); } From 59cc41c59385cde59906d2a74a16cf4bda0f190a Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:46:52 -0400 Subject: [PATCH 16/23] refactor: fix export statement for addCitationConsistencyWarnings and mergeMetadata --- src/services/githubImporter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index 6789b57..d9214c6 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -31,7 +31,7 @@ import { resolveContributorFallbackLimit, } from './githubImporterContributors.js'; import { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; -export { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js'; +export { addCitationConsistencyWarnings, mergeMetadata }; import { parseCargoToml, parsePackageJson, From 232efb83a1da665bd14b4f5c79291e370718cf0c Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Fri, 14 Aug 2026 16:49:01 -0400 Subject: [PATCH 17/23] refactor: rename parseJsonSafely to parseJson for consistency --- src/services/githubImporter.js | 4 ++-- src/services/githubImporterParsers.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/services/githubImporter.js b/src/services/githubImporter.js index d9214c6..13a57c9 100644 --- a/src/services/githubImporter.js +++ b/src/services/githubImporter.js @@ -414,12 +414,12 @@ export function parseCitationCff(text) { }; } -function parseJsonSafely(text) { +function parseJson(text) { return JSON.parse(text); } export function parseZenodoJson(text) { - const payload = parseJsonSafely(text); + const payload = parseJson(text); const creators = Array.isArray(payload.creators) ? payload.creators.map((creator) => normalizeAuthor(creator)).filter(Boolean) : []; diff --git a/src/services/githubImporterParsers.js b/src/services/githubImporterParsers.js index a985795..df8ff0d 100644 --- a/src/services/githubImporterParsers.js +++ b/src/services/githubImporterParsers.js @@ -12,7 +12,7 @@ const normalizeKeywords = utilNormalizeKeywords; const normalizeRepoUrl = utilNormalizeRepoUrl; const extractFirstMarkdownParagraph = utilExtractFirstMarkdownParagraph; -function parseJsonSafely(text) { +function parseJson(text) { return JSON.parse(text); } @@ -31,7 +31,7 @@ function extractPackageAuthors(payload) { } export function parsePackageJson(text) { - const payload = parseJsonSafely(text); + const payload = parseJson(text); return { title: cleanString(payload.name ?? ''), From e3983f730a7be409984aa5cfd34c7f0644754efd Mon Sep 17 00:00:00 2001 From: beanbean9339 Date: Sun, 23 Aug 2026 01:02:23 -0400 Subject: [PATCH 18/23] refactor: enhance metadata handling by tracking touched fields and extracting co-author names from commit messages --- src/App.jsx | 12 +- src/components/MetadataForm.jsx | 65 +++--- src/services/githubApi.js | 8 +- src/services/githubImporter.js | 29 ++- src/services/githubImporterContributors.js | 23 ++ src/services/githubImporterUtils.js | 6 +- src/styles.css | 7 + tests/services/githubImporter.test.js | 255 ++++++++++++++++++++- 8 files changed, 365 insertions(+), 40 deletions(-) diff --git a/src/App.jsx b/src/App.jsx index 01ce35e..064037d 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -389,6 +389,7 @@ export default function App() { const [copyState, setCopyState] = useState('idle'); const [orcidSuggestions, setOrcidSuggestions] = useState({}); const [exportNotice, setExportNotice] = useState({ kind: '', message: '', details: [] }); + const [touchedFields, setTouchedFields] = useState({}); const importRequestIdRef = useRef(0); const normalizedForm = useMemo(() => normalizeFormInput(form), [form]); const normalizedMetadata = useMemo(() => normalizeMetadata(normalizedForm), [normalizedForm]); @@ -496,6 +497,7 @@ export default function App() { function updateField(event) { const { name, value } = event.target; + setTouchedFields((current) => ({ ...current, [name]: true })); setForm((current) => ({ ...current, [name]: value })); } @@ -543,6 +545,11 @@ export default function App() { } function updateAuthorField(index, field, value) { + setTouchedFields((current) => ({ + ...current, + authors: true, + [`authors.${index}.${field}`]: true, + })); setForm((current) => ({ ...current, authors: current.authors.map((author, i) => (i === index ? { ...author, [field]: value } : author)), @@ -1033,8 +1040,8 @@ export default function App() {
Reviewed metadata loaded in editor.
    -
  • Done well: {healthScanSummary.pass}
    • -
  • Needs attention: {healthScanSummary.warning}
    • +
  • Warnings: {healthScanSummary.warning}
    • +
  • Passing checks: {healthScanSummary.pass}
  • Errors: {healthScanSummary.error}
@@ -1246,6 +1253,7 @@ export default function App() { licenseOptions={licenseOptions} grantSuggestions={grantSuggestions} errors={validationErrors} + touchedFields={touchedFields} orcidSuggestions={orcidSuggestions} updateField={updateField} appendGrantSuggestion={appendGrantSuggestion} diff --git a/src/components/MetadataForm.jsx b/src/components/MetadataForm.jsx index da40dee..87028cf 100644 --- a/src/components/MetadataForm.jsx +++ b/src/components/MetadataForm.jsx @@ -6,6 +6,7 @@ export function MetadataForm({ licenseOptions, grantSuggestions = [], errors = {}, + touchedFields = {}, orcidSuggestions = {}, updateField, appendGrantSuggestion, @@ -24,6 +25,10 @@ export function MetadataForm({ const totalAuthors = Array.isArray(form.authors) ? form.authors.length : 0; const visibleAuthorCount = showAllAuthors ? totalAuthors : Math.min(totalAuthors, AUTHORS_VISIBLE_BY_DEFAULT); const hiddenAuthorCount = Math.max(0, totalAuthors - visibleAuthorCount); + const hasTouchedField = (field) => Boolean(touchedFields[field]); + const hasTouchedAuthorField = (index, field) => Boolean(touchedFields[`authors.${index}.${field}`]); + const hasFieldError = (field) => Boolean(errors[field]); + const hasAuthorFieldError = (index, field) => Boolean(errors[`authorOrcid`]?.[index]); function toggleAuthorExpanded(index) { setExpandedAuthors((current) => ({ @@ -43,17 +48,17 @@ export function MetadataForm({

Start with what this work is and how people should reference it.

-