add parser .zenodo.json. Fixes #826

.gitignore

@@ -14,6 +14,7 @@ Scripts/*
 !**/composer.json
 !**/package.json
 !**/bower.json
+!**/.zenodo.json
 *.ttl
 !**/test_data/repositories/ecfo/ontology.ttl
 env_3.9/*

README.md

@@ -489,8 +489,8 @@ To run a classifier with an additional category or remove an existing one, a cor
 
 ## Metadata Support
 
-SOMEF supports the extraction and analysis of metadata in package files of several programming languages.  Current support includes: `setup.py` and `pyproject.toml` for Python, `pom.xml` for Java, `.gemspec` for Ruby, `DESCRIPTION` for R, `bower.json` for JavaScript, HTML or CSS, `.cabal` for Haskell, `cargo.toml` for RUST, `composer` for PHP, `.juliaProject.toml` for Julia , `AUTHORS`, `codemeta.json`, `publiccode.yml`, `dockerfile` and `citation.cff`
-This includes identifying dependencies, runtime requirements, and development tools specified in project configuration files.  
+SOMEF supports the extraction and analysis of metadata in package files of several programming languages.  Current support includes: `setup.py` and `pyproject.toml` for Python, `pom.xml` for Java, `.gemspec` for Ruby, `DESCRIPTION` for R, `bower.json` for JavaScript, HTML or CSS, `.cabal` for Haskell, `cargo.toml` for RUST, `composer` for PHP, `.juliaProject.toml` for Julia , `AUTHORS`, `codemeta.json`, `publiccode.yml`, `dockerfile` and `citation.cff` and `.zenodo.json`.
+This includes identifying dependencies, runtime requirements, and development tools specified in project configuration files. 
 
 ## Limitations
 

docs/supported_metadata_files.md

@@ -30,6 +30,7 @@ SOMEF can extract metadata from a wide range of files commonly found in software
 | `CITATION.cff` | YAML | Citation File Format for software citation metadata | [🔍](./citationcff.md) | [📄](https://citation-file-format.github.io/) | [v1.2.0](https://github.com/citation-file-format/citation-file-format) | [Example](https://github.com/citation-file-format/citation-file-format/blob/main/CITATION.cff) |
 | `publiccode.yml`       | YAML                      | YAML metadata file for public sector software projects| [🔍](./publiccode.md)| [📄](https://yml.publiccode.tools/)| [v0.3.0](https://yml.publiccode.tools/schema.html)| [Example](https://github.com/maykinmedia/objects-api/blob/master/publiccode.yaml) |
 | `environment.yml`       | YAML                      | Conda environment specification file declaring software dependencies for reproducible environments|  [🔍](./condaenvironment.md) | [📄](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually)| [Latest](https://github.com/conda/conda)| [Example](https://github.com/CompVis/stable-diffusion/blob/main/environment.yaml) |
+| `.zenodo.json` | JSON | Metadata file following the Zenodo metadata schema | [🔍](./zenodo.md) | [📄](https://help.zenodo.org/docs/github/describe-software/zenodo-json) | N/A | [Example](https://github.com/datacarpentry/cloud-genomics/blob/main/.zenodo.json) |
 
 
 > **Note:** The general principles behind metadata mapping in SOMEF are based on the [CodeMeta crosswalk](https://github.com/codemeta/codemeta/blob/master/crosswalk.csv) and the [CodeMeta JSON-LD context](https://github.com/codemeta/codemeta/blob/master/codemeta.jsonld).  

docs/zenodo.md

@@ -97,4 +97,4 @@ If the record does not point to a source repository, SOMEF stops with an error.
 
 With `--doi`, SOMEF downloads and analyzes the source repository found in the record, at the
 tag of the deposit if available, as it does in the git-based modes. The files of the Zenodo deposit
-itself are not downloaded or processed: they are only listed in the `download_url` category.
+itself are not downloaded or processed: they are only listed in the `download_url` category.

src/somef/parser/zenodo_parser.py

@@ -0,0 +1,162 @@
+import json
+import logging
+from pathlib import Path
+import re
+from ..process_results import Result
+from ..regular_expressions import detect_license_spdx
+from ..utils import constants
+
+
+def parse_zenodo_file(file_path, metadata_result: Result, source):
+    """
+    Parse a .zenodo.json file and extract relevant metadata.
+
+    Parameters
+    ----------
+    file_path: path of the zenodo file being analysed
+    metadata_result: metadata object where the metadata dictionary is kept
+    source: source of the package file (URL)
+
+    Returns
+    -------
+    metadata_result
+    """
+    try:
+        if Path(file_path).name.lower() in [".zenodo.json"]:
+            with open(file_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+
+            if not isinstance(data, dict):
+                return metadata_result
+
+            def add(category, value_dict):
+                metadata_result.add_result(category, value_dict, 1, constants.TECHNIQUE_CODE_CONFIG_PARSER, source)
+
+            if isinstance(data.get("title"), str) and data["title"].strip():
+                add(constants.CAT_NAME, {constants.PROP_VALUE: data["title"].strip(),constants.PROP_TYPE: constants.STRING})
+
+            description = data.get("description")
+            if isinstance(description, str):
+                description = re.sub(constants.REGEXP_CLEAN_HTML_TAGS, '', description).strip()
+                if description:
+                    add(constants.CAT_DESCRIPTION, {constants.PROP_VALUE: description,constants.PROP_TYPE: constants.STRING})
+
+            if data.get("version"):
+                add(constants.CAT_VERSION, {constants.PROP_VALUE: str(data["version"]).strip(), constants.PROP_TYPE: constants.STRING})
+
+            keywords = data.get("keywords")
+            if isinstance(keywords, str):
+                keywords = keywords.split(",")
+            if isinstance(keywords, list):
+                for kw in keywords:
+                    if isinstance(kw, str) and kw.strip():
+                        add(constants.CAT_KEYWORDS, {constants.PROP_VALUE: kw.strip(), constants.PROP_TYPE: constants.STRING})
+
+            license_info = parse_zenodo_license(data.get("license"))
+            if license_info:
+                license_data = {
+                    constants.PROP_VALUE: license_info["value"],
+                    constants.PROP_TYPE: constants.LICENSE,
+                }
+                if license_info.get("spdx_id"):
+                    license_data[constants.PROP_NAME] = license_info["name"]
+                    license_data[constants.PROP_SPDX_ID] = license_info["spdx_id"]
+                    license_data[constants.PROP_IDENTIFIER] = license_info["identifier"]
+                add(constants.CAT_LICENSE, license_data)
+
+            if data.get("publication_date"):
+                add(constants.CAT_DATE_PUBLISHED, {constants.PROP_VALUE: data["publication_date"], constants.PROP_TYPE: constants.STRING})
+
+            for person in data.get("creators") or []:
+                info = parse_zenodo_person(person)
+                if info:
+                    add(constants.CAT_AUTHORS, info)
+
+            for person in data.get("contributors") or []:
+                info = parse_zenodo_person(person)
+                if info:
+                    add(constants.CAT_CONTRIBUTORS, info)
+
+    except Exception as e:
+        logging.error(f"Error parsing zenodo.json from {file_path}: {str(e)}")
+
+    return metadata_result
+
+
+def clean_html(text):
+    """Remove HTML tags from a string, keeping readable text. The description in zenodo is usually in HTML format."""
+    if not isinstance(text, str):
+        return ""
+    soup = BeautifulSoup(text, "html.parser")
+    for br in soup.find_all("br"):
+        br.replace_with(" ")
+    for tag in soup.find_all(["p", "div", "li", "ul", "ol", "h1", "h2", "h3", "h4", "h5", "h6"]):
+        tag.append(" ")
+    return " ".join(soup.get_text().split())
+
+
+def parse_zenodo_license(license_data):
+    """
+    Parse the license field of a .zenodo.json file.
+    Zenodo uses either {"id": "mit"} or a plain string id.
+
+    Returns
+    -------
+    dict or None
+        Always has "value". If the license is recognised, it also has
+        name, spdx_id and identifier (SPDX URL).
+    """
+    if isinstance(license_data, dict):
+        license_id = license_data.get("id") or license_data.get("name")
+    elif isinstance(license_data, str):
+        license_id = license_data
+    else:
+        return None
+
+    if not isinstance(license_id, str) or not license_id.strip():
+        return None
+    license_id = license_id.strip()
+
+    license_info = {"value": license_id}
+    license_info_spdx = detect_license_spdx(license_id, "JSON")
+    if license_info_spdx:
+        license_info["name"] = license_info_spdx.get("name")
+        license_info["spdx_id"] = license_info_spdx.get("spdx_id")
+        license_info["identifier"] = license_info_spdx.get("identifier")
+    return license_info
+
+
+def parse_zenodo_person(person):
+    """
+    Parse an entry of 'creators' / 'contributors'.
+    Zenodo names usually come as "Family, Given".
+    """
+    if isinstance(person, str):
+        person = {"name": person}
+    if not isinstance(person, dict):
+        return None
+
+    name = (person.get("name") or "").strip()
+    if not name:
+        return None
+
+    info = {constants.PROP_TYPE: constants.AGENT}
+    if "," in name:
+        family, given = [p.strip() for p in name.split(",", 1)]
+        info[constants.PROP_VALUE] = f"{given} {family}".strip()
+        if given:
+            info[constants.PROP_GIVEN_NAME] = given
+        if family:
+            info[constants.PROP_LAST_NAME] = family
+    else:
+        info[constants.PROP_VALUE] = name
+    info[constants.PROP_NAME] = info[constants.PROP_VALUE]
+
+    if person.get("affiliation"):
+        info[constants.PROP_AFFILIATION] = person["affiliation"]
+    if person.get("orcid"):
+        orcid = person["orcid"].strip()
+        info[constants.PROP_IDENTIFIER] = orcid if orcid.startswith("http") else f"https://orcid.org/{orcid}"
+    return info
+
+

src/somef/process_files.py

@@ -27,6 +27,7 @@
 from .parser.codeowners_parser import parse_codeowners_file
 from .parser.conda_environment_parser import parse_conda_environment_file
 from .parser.setupcfg_parser import parse_setup_cfg
+from .parser.zenodo_parser import parse_zenodo_file
 from chardet import detect
 
 
@@ -341,8 +342,8 @@ def process_repository_files(repo_dir, metadata_result: Result, repo_type, owner
                             metadata_result = parse_conda_environment_file(os.path.join(dir_path, filename), metadata_result, build_file_url)
                         if filename.lower() == "setup.cfg":
                             metadata_result = parse_setup_cfg(os.path.join(dir_path, filename), metadata_result, build_file_url)
-                        # if filename.lower() == ".zenodo":
-                        #     metadata_result = parse_zenodo_file(os.path.join(dir_path, filename), metadata_result, build_file_url)
+                        if filename.lower() == ".zenodo.json":
+                            metadata_result = parse_zenodo_file(os.path.join(dir_path, filename), metadata_result, build_file_url)
                         parsed_build_files.add(filename.lower())
 
                 # if repo_type == constants.RepositoryType.GITLAB: 

src/somef/test/test_data/repositories/cloud-genomics/.zenodo.json

@@ -0,0 +1,69 @@
+{
+  "contributors": [
+    {
+      "type": "Editor",
+      "name": "Amanda Charbonneau",
+      "orcid": "0000-0001-7376-7702"
+    },
+    {
+      "type": "Editor",
+      "name": "Wendy Wong",
+      "orcid": "0000-0001-9850-3797"
+    },
+    {
+      "type": "Editor",
+      "name": "Anuj Guruacharya"
+    }
+  ],
+  "creators": [
+    {
+      "name": "Amanda Charbonneau",
+      "orcid": "0000-0001-7376-7702"
+    },
+    {
+      "name": "Zohaib Anwar",
+      "orcid": "0000-0001-8236-485X"
+    },
+    {
+      "name": "Wendy Wong",
+      "orcid": "0000-0001-9850-3797"
+    },
+    {
+      "name": "Darya P Vanichkina",
+      "orcid": "0000-0002-0406-164X"
+    },
+    {
+      "name": "Amanda Tan"
+    },
+    {
+      "name": "Christina Koch",
+      "orcid": "0000-0001-8600-8158"
+    },
+    {
+      "name": "Dinindu Senanayake"
+    },
+    {
+      "name": "LHurst-UoB"
+    },
+    {
+      "name": "PeterNICD"
+    },
+    {
+      "name": "Venkat Rajeev Reddy Malipeddi"
+    },
+    {
+      "name": "Serah Njambi",
+      "orcid": "0000-0002-7834-1038"
+    },
+    {
+      "name": "brynnelliott"
+    },
+    {
+      "name": "Daniel Kerchner",
+      "orcid": "0000-0002-5921-2193"
+    }
+  ],
+  "license": {
+    "id": "CC-BY-4.0"
+  }
+}

src/somef/test/test_data/repositories/epic/.zenodo.json

@@ -0,0 +1,41 @@
+{
+    "description": "The Elliptical Parcel-In-Cell (EPIC) model for fluid dynamics",
+    "access_right": "open",
+    "upload_type": "software",
+    "keywords": ["PIC",
+                 "elliptical parcels",
+                 "Lagrangian methods",
+                 "density stratified flows",
+                 "turbulent flows"],
+    "license": "BSD-3-Clause",
+    "creators": [
+      {
+          "affiliation": "University of St Andrews",
+          "name": "Frey, Matthias",
+          "orcid": "0000-0002-7842-0051"
+      },
+      {
+          "affiliation": "University of Leeds",
+          "name": "Böing, Steven",
+          "orcid": "0000-0003-3794-2563"
+      },
+      {
+          "affiliation": "University of St Andrews",
+          "name": "Dritschel, David",
+          "orcid": "0000-0001-6489-3395"
+      },
+      {
+          "affiliation": "Edinburgh Parallel Computing Centre (EPCC)",
+          "name": "Apóstolo, Rui",
+          "orcid": "0000-0003-1161-9098"
+      }
+    ],
+    "grants": [
+        {
+            "id": "10.13039/100014013::EP/T025301/1"
+        },
+        {
+            "id": "10.13039/100014013::EP/T025409/1"
+        }
+    ]
+}

src/somef/test/test_data/repositories/fake_zenodo/.zenodo.json

@@ -0,0 +1,35 @@
+{
+  "title": "Zenodo Test Software",
+  "description": "<p>Software to test the <b>zenodo</b> parser.</p>",
+  "version": "1.2.3",
+  "publication_date": "2024-05-17",
+  "upload_type": "software",
+  "access_right": "open",
+  "keywords": ["metadata", "research software", "zenodo"],
+  "license": {"id": "apache-2.0"},
+  "creators": [
+    {
+      "name": "Garijo, Daniel",
+      "affiliation": "Universidad Politécnica de Madrid",
+      "orcid": "0000-0003-0454-7145"
+    },
+    {
+      "name": "Maintainer Without Comma",
+      "orcid": "https://orcid.org/0000-0001-2345-6789"
+    },
+    {
+      "name": "Only Name"
+    }
+  ],
+  "contributors": [
+    {
+      "type": "Editor",
+      "name": "Doe, Jane",
+      "affiliation": "Example University"
+    },
+    {
+      "type": "Other",
+      "name": "Contributor Without Comma"
+    }
+  ]
+}