diff --git a/.gitignore b/.gitignore index 5c55fc87..e8f91dfe 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ Scripts/* !**/composer.json !**/package.json !**/bower.json +!**/.zenodo.json *.ttl !**/test_data/repositories/ecfo/ontology.ttl env_3.9/* diff --git a/README.md b/README.md index 8a6abd51..341ffa15 100644 --- a/README.md +++ b/README.md @@ -489,8 +489,8 @@ To run a classifier with an additional category or remove an existing one, a cor ## Metadata Support -SOMEF supports the extraction and analysis of metadata in package files of several programming languages. Current support includes: `setup.py` and `pyproject.toml` for Python, `pom.xml` for Java, `.gemspec` for Ruby, `DESCRIPTION` for R, `bower.json` for JavaScript, HTML or CSS, `.cabal` for Haskell, `cargo.toml` for RUST, `composer` for PHP, `.juliaProject.toml` for Julia , `AUTHORS`, `codemeta.json`, `publiccode.yml`, `dockerfile` and `citation.cff` -This includes identifying dependencies, runtime requirements, and development tools specified in project configuration files. +SOMEF supports the extraction and analysis of metadata in package files of several programming languages. Current support includes: `setup.py` and `pyproject.toml` for Python, `pom.xml` for Java, `.gemspec` for Ruby, `DESCRIPTION` for R, `bower.json` for JavaScript, HTML or CSS, `.cabal` for Haskell, `cargo.toml` for RUST, `composer` for PHP, `.juliaProject.toml` for Julia , `AUTHORS`, `codemeta.json`, `publiccode.yml`, `dockerfile` and `citation.cff` and `.zenodo.json`. +This includes identifying dependencies, runtime requirements, and development tools specified in project configuration files. ## Limitations diff --git a/docs/supported_metadata_files.md b/docs/supported_metadata_files.md index 403e5803..11557411 100644 --- a/docs/supported_metadata_files.md +++ b/docs/supported_metadata_files.md @@ -30,6 +30,7 @@ SOMEF can extract metadata from a wide range of files commonly found in software | `CITATION.cff` | YAML | Citation File Format for software citation metadata | [🔍](./citationcff.md) | [📄](https://citation-file-format.github.io/) | [v1.2.0](https://github.com/citation-file-format/citation-file-format) | [Example](https://github.com/citation-file-format/citation-file-format/blob/main/CITATION.cff) | | `publiccode.yml` | YAML | YAML metadata file for public sector software projects| [🔍](./publiccode.md)| [📄](https://yml.publiccode.tools/)| [v0.3.0](https://yml.publiccode.tools/schema.html)| [Example](https://github.com/maykinmedia/objects-api/blob/master/publiccode.yaml) | | `environment.yml` | YAML | Conda environment specification file declaring software dependencies for reproducible environments| [🔍](./condaenvironment.md) | [📄](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually)| [Latest](https://github.com/conda/conda)| [Example](https://github.com/CompVis/stable-diffusion/blob/main/environment.yaml) | +| `.zenodo.json` | JSON | Metadata file following the Zenodo metadata schema | [🔍](./zenodo.md) | [📄](https://help.zenodo.org/docs/github/describe-software/zenodo-json) | N/A | [Example](https://github.com/datacarpentry/cloud-genomics/blob/main/.zenodo.json) | > **Note:** The general principles behind metadata mapping in SOMEF are based on the [CodeMeta crosswalk](https://github.com/codemeta/codemeta/blob/master/crosswalk.csv) and the [CodeMeta JSON-LD context](https://github.com/codemeta/codemeta/blob/master/codemeta.jsonld). diff --git a/docs/zenodo.md b/docs/zenodo.md index 21a1dd31..d495492c 100644 --- a/docs/zenodo.md +++ b/docs/zenodo.md @@ -97,4 +97,4 @@ If the record does not point to a source repository, SOMEF stops with an error. With `--doi`, SOMEF downloads and analyzes the source repository found in the record, at the tag of the deposit if available, as it does in the git-based modes. The files of the Zenodo deposit -itself are not downloaded or processed: they are only listed in the `download_url` category. \ No newline at end of file +itself are not downloaded or processed: they are only listed in the `download_url` category. diff --git a/src/somef/parser/zenodo_parser.py b/src/somef/parser/zenodo_parser.py new file mode 100644 index 00000000..34174c88 --- /dev/null +++ b/src/somef/parser/zenodo_parser.py @@ -0,0 +1,162 @@ +import json +import logging +from pathlib import Path +import re +from ..process_results import Result +from ..regular_expressions import detect_license_spdx +from ..utils import constants + + +def parse_zenodo_file(file_path, metadata_result: Result, source): + """ + Parse a .zenodo.json file and extract relevant metadata. + + Parameters + ---------- + file_path: path of the zenodo file being analysed + metadata_result: metadata object where the metadata dictionary is kept + source: source of the package file (URL) + + Returns + ------- + metadata_result + """ + try: + if Path(file_path).name.lower() in [".zenodo.json"]: + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + + if not isinstance(data, dict): + return metadata_result + + def add(category, value_dict): + metadata_result.add_result(category, value_dict, 1, constants.TECHNIQUE_CODE_CONFIG_PARSER, source) + + if isinstance(data.get("title"), str) and data["title"].strip(): + add(constants.CAT_NAME, {constants.PROP_VALUE: data["title"].strip(),constants.PROP_TYPE: constants.STRING}) + + description = data.get("description") + if isinstance(description, str): + description = re.sub(constants.REGEXP_CLEAN_HTML_TAGS, '', description).strip() + if description: + add(constants.CAT_DESCRIPTION, {constants.PROP_VALUE: description,constants.PROP_TYPE: constants.STRING}) + + if data.get("version"): + add(constants.CAT_VERSION, {constants.PROP_VALUE: str(data["version"]).strip(), constants.PROP_TYPE: constants.STRING}) + + keywords = data.get("keywords") + if isinstance(keywords, str): + keywords = keywords.split(",") + if isinstance(keywords, list): + for kw in keywords: + if isinstance(kw, str) and kw.strip(): + add(constants.CAT_KEYWORDS, {constants.PROP_VALUE: kw.strip(), constants.PROP_TYPE: constants.STRING}) + + license_info = parse_zenodo_license(data.get("license")) + if license_info: + license_data = { + constants.PROP_VALUE: license_info["value"], + constants.PROP_TYPE: constants.LICENSE, + } + if license_info.get("spdx_id"): + license_data[constants.PROP_NAME] = license_info["name"] + license_data[constants.PROP_SPDX_ID] = license_info["spdx_id"] + license_data[constants.PROP_IDENTIFIER] = license_info["identifier"] + add(constants.CAT_LICENSE, license_data) + + if data.get("publication_date"): + add(constants.CAT_DATE_PUBLISHED, {constants.PROP_VALUE: data["publication_date"], constants.PROP_TYPE: constants.STRING}) + + for person in data.get("creators") or []: + info = parse_zenodo_person(person) + if info: + add(constants.CAT_AUTHORS, info) + + for person in data.get("contributors") or []: + info = parse_zenodo_person(person) + if info: + add(constants.CAT_CONTRIBUTORS, info) + + except Exception as e: + logging.error(f"Error parsing zenodo.json from {file_path}: {str(e)}") + + return metadata_result + + +def clean_html(text): + """Remove HTML tags from a string, keeping readable text. The description in zenodo is usually in HTML format.""" + if not isinstance(text, str): + return "" + soup = BeautifulSoup(text, "html.parser") + for br in soup.find_all("br"): + br.replace_with(" ") + for tag in soup.find_all(["p", "div", "li", "ul", "ol", "h1", "h2", "h3", "h4", "h5", "h6"]): + tag.append(" ") + return " ".join(soup.get_text().split()) + + +def parse_zenodo_license(license_data): + """ + Parse the license field of a .zenodo.json file. + Zenodo uses either {"id": "mit"} or a plain string id. + + Returns + ------- + dict or None + Always has "value". If the license is recognised, it also has + name, spdx_id and identifier (SPDX URL). + """ + if isinstance(license_data, dict): + license_id = license_data.get("id") or license_data.get("name") + elif isinstance(license_data, str): + license_id = license_data + else: + return None + + if not isinstance(license_id, str) or not license_id.strip(): + return None + license_id = license_id.strip() + + license_info = {"value": license_id} + license_info_spdx = detect_license_spdx(license_id, "JSON") + if license_info_spdx: + license_info["name"] = license_info_spdx.get("name") + license_info["spdx_id"] = license_info_spdx.get("spdx_id") + license_info["identifier"] = license_info_spdx.get("identifier") + return license_info + + +def parse_zenodo_person(person): + """ + Parse an entry of 'creators' / 'contributors'. + Zenodo names usually come as "Family, Given". + """ + if isinstance(person, str): + person = {"name": person} + if not isinstance(person, dict): + return None + + name = (person.get("name") or "").strip() + if not name: + return None + + info = {constants.PROP_TYPE: constants.AGENT} + if "," in name: + family, given = [p.strip() for p in name.split(",", 1)] + info[constants.PROP_VALUE] = f"{given} {family}".strip() + if given: + info[constants.PROP_GIVEN_NAME] = given + if family: + info[constants.PROP_LAST_NAME] = family + else: + info[constants.PROP_VALUE] = name + info[constants.PROP_NAME] = info[constants.PROP_VALUE] + + if person.get("affiliation"): + info[constants.PROP_AFFILIATION] = person["affiliation"] + if person.get("orcid"): + orcid = person["orcid"].strip() + info[constants.PROP_IDENTIFIER] = orcid if orcid.startswith("http") else f"https://orcid.org/{orcid}" + return info + + diff --git a/src/somef/process_files.py b/src/somef/process_files.py index f2253e8a..4b80b4ef 100644 --- a/src/somef/process_files.py +++ b/src/somef/process_files.py @@ -27,6 +27,7 @@ from .parser.codeowners_parser import parse_codeowners_file from .parser.conda_environment_parser import parse_conda_environment_file from .parser.setupcfg_parser import parse_setup_cfg +from .parser.zenodo_parser import parse_zenodo_file from chardet import detect @@ -341,8 +342,8 @@ def process_repository_files(repo_dir, metadata_result: Result, repo_type, owner metadata_result = parse_conda_environment_file(os.path.join(dir_path, filename), metadata_result, build_file_url) if filename.lower() == "setup.cfg": metadata_result = parse_setup_cfg(os.path.join(dir_path, filename), metadata_result, build_file_url) - # if filename.lower() == ".zenodo": - # metadata_result = parse_zenodo_file(os.path.join(dir_path, filename), metadata_result, build_file_url) + if filename.lower() == ".zenodo.json": + metadata_result = parse_zenodo_file(os.path.join(dir_path, filename), metadata_result, build_file_url) parsed_build_files.add(filename.lower()) # if repo_type == constants.RepositoryType.GITLAB: diff --git a/src/somef/test/test_data/repositories/cloud-genomics/.zenodo.json b/src/somef/test/test_data/repositories/cloud-genomics/.zenodo.json new file mode 100644 index 00000000..28b52478 --- /dev/null +++ b/src/somef/test/test_data/repositories/cloud-genomics/.zenodo.json @@ -0,0 +1,69 @@ +{ + "contributors": [ + { + "type": "Editor", + "name": "Amanda Charbonneau", + "orcid": "0000-0001-7376-7702" + }, + { + "type": "Editor", + "name": "Wendy Wong", + "orcid": "0000-0001-9850-3797" + }, + { + "type": "Editor", + "name": "Anuj Guruacharya" + } + ], + "creators": [ + { + "name": "Amanda Charbonneau", + "orcid": "0000-0001-7376-7702" + }, + { + "name": "Zohaib Anwar", + "orcid": "0000-0001-8236-485X" + }, + { + "name": "Wendy Wong", + "orcid": "0000-0001-9850-3797" + }, + { + "name": "Darya P Vanichkina", + "orcid": "0000-0002-0406-164X" + }, + { + "name": "Amanda Tan" + }, + { + "name": "Christina Koch", + "orcid": "0000-0001-8600-8158" + }, + { + "name": "Dinindu Senanayake" + }, + { + "name": "LHurst-UoB" + }, + { + "name": "PeterNICD" + }, + { + "name": "Venkat Rajeev Reddy Malipeddi" + }, + { + "name": "Serah Njambi", + "orcid": "0000-0002-7834-1038" + }, + { + "name": "brynnelliott" + }, + { + "name": "Daniel Kerchner", + "orcid": "0000-0002-5921-2193" + } + ], + "license": { + "id": "CC-BY-4.0" + } +} \ No newline at end of file diff --git a/src/somef/test/test_data/repositories/epic/.zenodo.json b/src/somef/test/test_data/repositories/epic/.zenodo.json new file mode 100644 index 00000000..426a63a7 --- /dev/null +++ b/src/somef/test/test_data/repositories/epic/.zenodo.json @@ -0,0 +1,41 @@ +{ + "description": "The Elliptical Parcel-In-Cell (EPIC) model for fluid dynamics", + "access_right": "open", + "upload_type": "software", + "keywords": ["PIC", + "elliptical parcels", + "Lagrangian methods", + "density stratified flows", + "turbulent flows"], + "license": "BSD-3-Clause", + "creators": [ + { + "affiliation": "University of St Andrews", + "name": "Frey, Matthias", + "orcid": "0000-0002-7842-0051" + }, + { + "affiliation": "University of Leeds", + "name": "Böing, Steven", + "orcid": "0000-0003-3794-2563" + }, + { + "affiliation": "University of St Andrews", + "name": "Dritschel, David", + "orcid": "0000-0001-6489-3395" + }, + { + "affiliation": "Edinburgh Parallel Computing Centre (EPCC)", + "name": "Apóstolo, Rui", + "orcid": "0000-0003-1161-9098" + } + ], + "grants": [ + { + "id": "10.13039/100014013::EP/T025301/1" + }, + { + "id": "10.13039/100014013::EP/T025409/1" + } + ] +} \ No newline at end of file diff --git a/src/somef/test/test_data/repositories/fake_zenodo/.zenodo.json b/src/somef/test/test_data/repositories/fake_zenodo/.zenodo.json new file mode 100644 index 00000000..2003b64c --- /dev/null +++ b/src/somef/test/test_data/repositories/fake_zenodo/.zenodo.json @@ -0,0 +1,35 @@ +{ + "title": "Zenodo Test Software", + "description": "

Software to test the zenodo parser.

", + "version": "1.2.3", + "publication_date": "2024-05-17", + "upload_type": "software", + "access_right": "open", + "keywords": ["metadata", "research software", "zenodo"], + "license": {"id": "apache-2.0"}, + "creators": [ + { + "name": "Garijo, Daniel", + "affiliation": "Universidad Politécnica de Madrid", + "orcid": "0000-0003-0454-7145" + }, + { + "name": "Maintainer Without Comma", + "orcid": "https://orcid.org/0000-0001-2345-6789" + }, + { + "name": "Only Name" + } + ], + "contributors": [ + { + "type": "Editor", + "name": "Doe, Jane", + "affiliation": "Example University" + }, + { + "type": "Other", + "name": "Contributor Without Comma" + } + ] +} diff --git a/src/somef/test/test_zenodo_parser.py b/src/somef/test/test_zenodo_parser.py new file mode 100644 index 00000000..dd5413f9 --- /dev/null +++ b/src/somef/test/test_zenodo_parser.py @@ -0,0 +1,135 @@ +import unittest +import os +from pathlib import Path +from somef.process_results import Result +from somef.parser.zenodo_parser import parse_zenodo_file + +test_data_path = str(Path(__file__).parent / "test_data") + os.path.sep +test_data_repositories = str(Path(__file__).parent / "test_data" / "repositories") + os.path.sep + + +class TestZenodoParser(unittest.TestCase): + + def test_issue_826_creators_contributors_license(self): + """Checks if this repository has properties extracted from .zenodo.json. datacarpentry/cloud-genomics repository""" + zenodo_file_path = test_data_repositories + "cloud-genomics" + os.path.sep + ".zenodo.json" + result = Result() + + metadata_result = parse_zenodo_file( + zenodo_file_path, + result, + zenodo_file_path + ) + + authors = metadata_result.results.get("author", []) + author_names = [entry["result"]["value"] for entry in authors] + self.assertEqual(len(authors), 13) + self.assertIn("Amanda Charbonneau", author_names) + self.assertIn("Serah Njambi", author_names) + for entry in authors: + self.assertEqual(entry["technique"], "code_parser") + + serah = next(a for a in authors if a["result"]["value"] == "Serah Njambi") + self.assertEqual(serah["result"]["identifier"], "https://orcid.org/0000-0002-7834-1038") + + contributors = metadata_result.results.get("contributor", []) + contributor_names = [entry["result"]["value"] for entry in contributors] + self.assertEqual(contributor_names, ["Amanda Charbonneau", "Wendy Wong", "Anuj Guruacharya"]) + + licenses = metadata_result.results.get("license", []) + self.assertEqual(len(licenses), 1) + self.assertEqual(licenses[0]["result"]["value"], "CC-BY-4.0") + self.assertEqual(licenses[0]["result"]["spdx_id"], "CC-BY-4.0") + + def test_issue_826_description_keywords_names(self): + """Checks description, keywords, license and creators with 'Family, Given' names. EPIC-model/epic repository""" + zenodo_file_path = test_data_repositories + "epic" + os.path.sep + ".zenodo.json" + result = Result() + + metadata_result = parse_zenodo_file( + zenodo_file_path, + result, + zenodo_file_path + ) + + descriptions = metadata_result.results.get("description", []) + self.assertEqual(len(descriptions), 1) + self.assertEqual(descriptions[0]["result"]["value"], + "The Elliptical Parcel-In-Cell (EPIC) model for fluid dynamics") + + keywords = [entry["result"]["value"] for entry in metadata_result.results.get("keywords", [])] + self.assertEqual(len(keywords), 5) + self.assertIn("Lagrangian methods", keywords) + + licenses = metadata_result.results.get("license", []) + self.assertEqual(licenses[0]["result"]["value"], "BSD-3-Clause") + self.assertEqual(licenses[0]["result"]["spdx_id"], "BSD-3-Clause") + + authors = metadata_result.results.get("author", []) + self.assertEqual(len(authors), 4) + frey = authors[0]["result"] + self.assertEqual(frey["value"], "Matthias Frey") + self.assertEqual(frey["given_name"], "Matthias") + self.assertEqual(frey["last_name"], "Frey") + self.assertEqual(frey["affiliation"], "University of St Andrews") + self.assertEqual(frey["identifier"], "https://orcid.org/0000-0002-7842-0051") + + + def test_issue_826_all_supported_fields(self): + """Synthetic .zenodo.json covering every field supported by the parser""" + zenodo_file_path = test_data_repositories + "fake_zenodo" + os.path.sep + ".zenodo.json" + result = Result() + + metadata_result = parse_zenodo_file( + zenodo_file_path, + result, + zenodo_file_path + ) + results = metadata_result.results + + self.assertEqual(results["name"][0]["result"]["value"], "Zenodo Test Software") + self.assertEqual(results["version"][0]["result"]["value"], "1.2.3") + self.assertEqual(results["date_published"][0]["result"]["value"], "2024-05-17") + + # HTML must be removed from the description + description = results["description"][0]["result"]["value"] + self.assertEqual(description, "Software to test the zenodo parser.") + self.assertNotIn("<", description) + + keywords = [entry["result"]["value"] for entry in results["keywords"]] + self.assertEqual(keywords, ["metadata", "research software", "zenodo"]) + + # Zenodo ids are lowercase ("apache-2.0"); value is kept, spdx_id is the canonical one + license_result = results["license"][0]["result"] + self.assertEqual(license_result["value"], "apache-2.0") + self.assertEqual(license_result["spdx_id"], "Apache-2.0") + self.assertEqual(license_result["identifier"], "https://spdx.org/licenses/Apache-2.0") + + authors = [entry["result"] for entry in results["author"]] + self.assertEqual([a["value"] for a in authors], + ["Daniel Garijo", "Maintainer Without Comma", "Only Name"]) + # "Family, Given" is split + self.assertEqual(authors[0]["given_name"], "Daniel") + self.assertEqual(authors[0]["last_name"], "Garijo") + self.assertEqual(authors[0]["affiliation"], "Universidad Politécnica de Madrid") + self.assertEqual(authors[0]["identifier"], "https://orcid.org/0000-0003-0454-7145") + # names without a comma are not split + self.assertNotIn("given_name", authors[1]) + self.assertNotIn("last_name", authors[1]) + # an ORCID already given as URL is not prefixed twice + self.assertEqual(authors[1]["identifier"], "https://orcid.org/0000-0001-2345-6789") + self.assertNotIn("identifier", authors[2]) + + contributors = [entry["result"] for entry in results["contributor"]] + self.assertEqual([c["value"] for c in contributors], ["Jane Doe", "Contributor Without Comma"]) + self.assertEqual(contributors[0]["given_name"], "Jane") + self.assertEqual(contributors[0]["affiliation"], "Example University") + + for category in ["name", "description", "version", "keywords", "license", + "date_published", "author", "contributor"]: + for entry in results[category]: + self.assertEqual(entry["technique"], "code_parser") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/somef/utils/constants.py b/src/somef/utils/constants.py index 20a744f8..7db63af2 100644 --- a/src/somef/utils/constants.py +++ b/src/somef/utils/constants.py @@ -102,6 +102,7 @@ REGEXP_LGPL1 = r'(?i)gnu\s+lesser\s+general\s+public\s+license\s*,?\s*version\s*1\.0' REGEXP_MPL2 = r'(?i)mozilla\s+public\s+license\s*,?\s*version\s*2\.0' REGEXP_UNLICENSE = r'(?i)the\s+unlicense' +REGEXP_CCBY4 = r'(?i)(creative\s+commons\s+attribution\s+4\.0(?:\s+international)?|cc[\s-]by[\s-]4\.0)' # detect choosealicense in badges REGEXP_CHOOSE_LICENSE = r'choosealicense\.com/licenses/([^/\s]+)' @@ -141,6 +142,7 @@ "BSD 3-Clause": {"regex": REGEXP_BSD3, "spdx_id": "BSD-3-Clause"}, "Boost Software License 1.0": {"regex": REGEXP_BOOST, "spdx_id": "BSL-1.0"}, "Creative Commons Zero v1.0": {"regex": REGEXP_CC0, "spdx_id": "CC0-1.0"}, + "Creative Commons Attribution 4.0 International": {"regex": REGEXP_CCBY4, "spdx_id": "CC-BY-4.0"}, "Eclipse Public License 2.0": {"regex": REGEXP_EPL2, "spdx_id": "EPL-2.0"}, "GNU Affero General Public License v3.0": {"regex": REGEXP_AGPL3, "spdx_id": "AGPL-3.0"}, "GNU General Public License v2": {"regex": REGEXP_GPL2, "spdx_id": "GPL-2.0"},