diff --git a/build.sbt b/build.sbt index e087e18..f9a9f61 100644 --- a/build.sbt +++ b/build.sbt @@ -29,6 +29,9 @@ buildInfoPackage := "org.monarchinitiative.dosdp.cli" val zioVersion = "2.1.26" val zioLoggingVersion = "2.5.3" +val catsVersion = "2.13.0" +val circeVersion = "0.14.15" +val circeYamlVersion = "0.16.1" libraryDependencies ++= { Seq( @@ -36,25 +39,19 @@ libraryDependencies ++= { "com.github.alexarchambault" %% "case-app" % "2.0.6", "net.sourceforge.owlapi" % "owlapi-distribution" % "4.5.29", "org.phenoscape" %% "scowl" % "1.4.1", - "org.phenoscape" %% "owlet" % "1.9" exclude("org.slf4j", "slf4j-log4j12"), - "org.semanticweb.elk" % "elk-owlapi" % "0.4.3" exclude("org.slf4j", "slf4j-log4j12"), + "io.github.liveontologies" % "elk-owlapi" % "0.6.0" + exclude("net.sourceforge.owlapi", "owlapi-apibinding") + exclude("net.sourceforge.owlapi", "owlapi-api") + exclude("net.sourceforge.owlapi", "owlapi-impl"), "net.sourceforge.owlapi" % "org.semanticweb.hermit" % "1.4.3.456", "net.sourceforge.owlapi" % "jfact" % "4.0.4", - "org.geneontology" %% "owl-diff" % "1.3.0", - "io.circe" %% "circe-yaml" % "0.14.2", - "io.circe" %% "circe-core" % "0.14.15", - "io.circe" %% "circe-generic" % "0.14.15", - "io.circe" %% "circe-parser" % "0.14.15", - "org.obolibrary.robot" % "robot-core" % "1.8.4" - exclude("ch.qos.logback", "logback-classic") - exclude("org.slf4j", "slf4j-log4j12") - exclude("org.geneontology", "whelk_2.12") - exclude("org.geneontology", "whelk-owlapi_2.12") - exclude("org.geneontology", "owl-diff_2.12"), - "com.github.pathikrit" %% "better-files" % "3.9.2", - "org.apache.jena" % "apache-jena-libs" % "4.10.0" exclude("org.slf4j", "slf4j-log4j12"), + "org.typelevel" %% "cats-core" % catsVersion, + "io.circe" %% "circe-yaml" % circeYamlVersion, + "io.circe" %% "circe-core" % circeVersion, + "io.circe" %% "circe-generic" % circeVersion, + "org.apache.jena" % "jena-arq" % "4.10.0", + "org.apache.jena" % "jena-core" % "4.10.0", "com.github.tototoshi" %% "scala-csv" % "2.0.0", - "commons-codec" % "commons-codec" % "1.17.2", "dev.zio" %% "zio-logging" % zioLoggingVersion, "dev.zio" %% "zio-logging-slf4j2-bridge" % zioLoggingVersion, "dev.zio" %% "zio-test" % zioVersion % Test, diff --git a/src/main/scala/org/monarchinitiative/dosdp/CatalogXmlIRIMapper.scala b/src/main/scala/org/monarchinitiative/dosdp/CatalogXmlIRIMapper.scala new file mode 100644 index 0000000..a16b65f --- /dev/null +++ b/src/main/scala/org/monarchinitiative/dosdp/CatalogXmlIRIMapper.scala @@ -0,0 +1,121 @@ +package org.monarchinitiative.dosdp + +import org.semanticweb.owlapi.model.{IRI, OWLOntologyIRIMapper} +import org.xml.sax.Attributes +import org.xml.sax.helpers.DefaultHandler + +import java.io.{File, FileInputStream, InputStream} +import java.net.URL +import java.util.HashMap +import javax.xml.parsers.SAXParserFactory +import scala.util.control.NonFatal + +/** + * OWL ontology IRI mapper backed by OASIS-style catalog XML `uri` entries. + * + * Adapted from ROBOT's `org.obolibrary.robot.CatalogXmlIRIMapper` and + * `CatalogElementHandler` implementations (BSD-3-Clause). Keeping this local + * avoids depending on all of `robot-core` for one small mapper. + */ +final class CatalogXmlIRIMapper private (mappings: java.util.Map[IRI, IRI]) extends OWLOntologyIRIMapper { + + def this(catalogFile: String) = + this(CatalogXmlIRIMapper.parseCatalogFile(new File(catalogFile).getAbsoluteFile)) + + def this(catalogFile: File) = + this(CatalogXmlIRIMapper.parseCatalogFile(catalogFile.getAbsoluteFile)) + + def this(catalogFile: File, parentFolder: File) = + this(CatalogXmlIRIMapper.parseCatalogXml(new FileInputStream(catalogFile), Option(parentFolder))) + + def this(catalogIRI: IRI) = + this(CatalogXmlIRIMapper.parseCatalogXml(catalogIRI.toURI.toURL)) + + def this(catalogURL: URL) = + this(CatalogXmlIRIMapper.parseCatalogXml(catalogURL)) + + def this(catalogURL: URL, parentFolder: File) = + this(CatalogXmlIRIMapper.parseCatalogXml(catalogURL.openStream(), Option(parentFolder))) + + override def getDocumentIRI(ontologyIRI: IRI): IRI = + mappings.get(ontologyIRI) + +} + +private object CatalogXmlIRIMapper { + + private def parseCatalogFile(catalogFile: File): java.util.Map[IRI, IRI] = + parseCatalogXml(new FileInputStream(catalogFile), Option(catalogFile.getParentFile)) + + private def parseCatalogXml(catalogURL: URL): java.util.Map[IRI, IRI] = + if (catalogURL.getProtocol == "file") { + val catalogFile = new File(catalogURL.toURI) + parseCatalogXml(new FileInputStream(catalogFile), Option(catalogFile.getParentFile)) + } else parseCatalogXml(catalogURL.openStream(), None) + + private def parseCatalogXml(inputStream: InputStream, parentFolder: Option[File]): java.util.Map[IRI, IRI] = { + val mappings = new HashMap[IRI, IRI]() + try { + val factory = SAXParserFactory.newInstance() + factory.setValidating(false) + factory.setNamespaceAware(true) + disableExternalXml(factory) + val saxParser = factory.newSAXParser() + saxParser.parse(inputStream, new CatalogElementHandler(parentFolder, mappings)) + mappings + } finally inputStream.close() + } + + private def disableExternalXml(factory: SAXParserFactory): Unit = { + setFeatureIfSupported(factory, "http://xml.org/sax/features/external-general-entities", false) + setFeatureIfSupported(factory, "http://xml.org/sax/features/external-parameter-entities", false) + setFeatureIfSupported(factory, "http://apache.org/xml/features/nonvalidating/load-external-dtd", false) + try factory.setXIncludeAware(false) + catch { + case NonFatal(_) => () + } + } + + private def setFeatureIfSupported(factory: SAXParserFactory, feature: String, value: Boolean): Unit = + try factory.setFeature(feature, value) + catch { + case NonFatal(_) => () + } + + private final class CatalogElementHandler(parentFolder: Option[File], mappings: java.util.Map[IRI, IRI]) extends DefaultHandler { + + override def startElement(uri: String, localName: String, qName: String, attributes: Attributes): Unit = { + if (elementName(localName, qName) != "uri") return + + val fromString = attributeValue(attributes, "name") + val toString = attributeValue(attributes, "uri") + if ((fromString == null) || (toString == null)) return + + mappedIRI(toString).foreach { toIRI => + mappings.put(IRI.create(fromString), toIRI) + } + } + + private def elementName(localName: String, qName: String): String = + if (localName != null && localName.nonEmpty) localName + else Option(qName).fold("")(_.split(':').last) + + private def attributeValue(attributes: Attributes, name: String): String = + Option(attributes.getValue("", name)).orElse(Option(attributes.getValue(name))).orNull + + private def mappedIRI(value: String): Option[IRI] = + parentFolder.filter(_ => !value.contains(":")) match { + case Some(parent) => + try { + val file = new File(value) + val resolved = if (file.isAbsolute) file else new File(parent, value) + Some(IRI.create(resolved.getCanonicalFile)) + } catch { + case NonFatal(_) => None + } + case None => Some(IRI.create(value)) + } + + } + +} diff --git a/src/main/scala/org/monarchinitiative/dosdp/DOSDP.scala b/src/main/scala/org/monarchinitiative/dosdp/DOSDP.scala index a7e23bd..8fd6738 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/DOSDP.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/DOSDP.scala @@ -3,10 +3,12 @@ package org.monarchinitiative.dosdp import io.circe._ import io.circe.generic.auto._ import io.circe.syntax._ -import org.apache.commons.codec.digest.DigestUtils import org.phenoscape.scowl._ import org.semanticweb.owlapi.model.{IRI, OWLAnnotationProperty} +import java.nio.charset.StandardCharsets +import java.security.MessageDigest + /** * Basic data model for DOSDP schema, for serializing to/from JSON. */ @@ -77,10 +79,16 @@ object DOSDP { case MultiValue(values) => values.toSeq.sorted.mkString("|") } }.mkString("&") - val hash = DigestUtils.sha1Hex(text) + val hash = sha1Hex(text) IRI.create(s"$pattern#$hash") } + private def sha1Hex(text: String): String = + MessageDigest.getInstance("SHA-1") + .digest(text.getBytes(StandardCharsets.UTF_8)) + .map(byte => f"${byte & 0xff}%02x") + .mkString + } trait PrintfText { @@ -366,4 +374,3 @@ final case class Join(sep: String) * will be used to generate additional annotations. */ final case class Permutation(`var`: String, annotationProperties: List[String]) - diff --git a/src/main/scala/org/monarchinitiative/dosdp/ManchesterSyntaxOWLObjectRenderer.scala b/src/main/scala/org/monarchinitiative/dosdp/ManchesterSyntaxOWLObjectRenderer.scala new file mode 100644 index 0000000..54d0e3b --- /dev/null +++ b/src/main/scala/org/monarchinitiative/dosdp/ManchesterSyntaxOWLObjectRenderer.scala @@ -0,0 +1,101 @@ +package org.monarchinitiative.dosdp + +import org.semanticweb.owlapi.apibinding.OWLManager +import org.semanticweb.owlapi.io.OWLObjectRenderer +import org.semanticweb.owlapi.manchestersyntax.renderer.ManchesterOWLSyntaxObjectRenderer +import org.semanticweb.owlapi.model.{IRI, OWLDataVisitor, OWLLiteral, OWLObject} +import org.semanticweb.owlapi.util.{ShortFormProvider, SimpleShortFormProvider} +import org.semanticweb.owlapi.vocab.XSDVocabulary + +import java.io.{StringWriter, Writer} + +/** + * Manchester syntax renderer that applies short forms to bare IRIs. + * + * Adapted from owl-diff's `ManchesterSyntaxOWLObjectRenderer`. The + * literal escaping is implemented locally so this + * project does not need owl-diff's transitive `commons-text` dependency. + */ +class ManchesterSyntaxOWLObjectRenderer extends OWLObjectRenderer { + + import ManchesterSyntaxOWLObjectRenderer._ + + private object WriterDelegate extends Writer { + + private var delegate = new StringWriter() + + def reset(): Unit = delegate = new StringWriter() + + override def toString: String = delegate.getBuffer.toString + + override def close(): Unit = delegate.close() + + override def flush(): Unit = delegate.flush() + + override def write(cbuf: Array[Char], off: Int, len: Int): Unit = delegate.write(cbuf, off, len) + + } + + private var renderer: ManchesterOWLSyntaxObjectRenderer = new BetterIRIRenderer(WriterDelegate, new SimpleShortFormProvider()) + + override def render(obj: OWLObject): String = synchronized { + WriterDelegate.reset() + obj.accept(renderer) + WriterDelegate.toString + } + + override def setShortFormProvider(shortFormProvider: ShortFormProvider): Unit = synchronized { + renderer = new BetterIRIRenderer(WriterDelegate, shortFormProvider) + } + +} + +object ManchesterSyntaxOWLObjectRenderer { + + private val factory = OWLManager.getOWLDataFactory + + private def escapeHtmlText(text: String): String = + text.flatMap { + case '&' => "&" + case '<' => "<" + case '>' => ">" + case '"' => """ + case char => char.toString + } + + class BetterIRIRenderer(writer: Writer, entityShortFormProvider: ShortFormProvider) extends ManchesterOWLSyntaxObjectRenderer(writer, entityShortFormProvider) { + + override def visit(iri: IRI): Unit = visit(factory.getOWLClass(iri)) + + override def visit(node: OWLLiteral): Unit = { + + // xsd:decimal is the default datatype for literal forms like "33.3" + // with no specified datatype + if (XSDVocabulary.DECIMAL.getIRI.equals(node.getDatatype.getIRI)) { + write(node.getLiteral) + } else if (node.getDatatype.isFloat) { + write(node.getLiteral) + write("f") + } else if (node.getDatatype.isInteger) { + write(node.getLiteral) + } else if (node.getDatatype.isBoolean) { + write(node.getLiteral) + } else { + pushTab(getIndent) + write("\"") + write(escapeHtmlText(node.getLiteral)) + write("\"") + if (node.hasLang()) { + write("@") + write(node.getLang) + } else if (!node.isRDFPlainLiteral) { + write("^^") + node.getDatatype.accept(this: OWLDataVisitor) + } + popTab() + } + } + + } + +} diff --git a/src/main/scala/org/monarchinitiative/dosdp/MarkdownLinkShortFormProvider.scala b/src/main/scala/org/monarchinitiative/dosdp/MarkdownLinkShortFormProvider.scala new file mode 100644 index 0000000..f7bc4fa --- /dev/null +++ b/src/main/scala/org/monarchinitiative/dosdp/MarkdownLinkShortFormProvider.scala @@ -0,0 +1,23 @@ +package org.monarchinitiative.dosdp + +import org.semanticweb.owlapi.model.OWLEntity +import org.semanticweb.owlapi.util.ShortFormProvider + +/** + * Short-form provider that renders OWL entities as Markdown links. + * + * Adapted from owl-diff's `MarkdownLinkShortFormProvider`. Keeping + * this local avoids depending on all of `owl-diff` for + * one small docs-rendering helper. + */ +class MarkdownLinkShortFormProvider(labelProvider: ShortFormProvider) extends ShortFormProvider { + + override def getShortForm(entity: OWLEntity): String = { + val label = labelProvider.getShortForm(entity) + val iri = entity.getIRI.toString + s"[$label]($iri)" + } + + override def dispose(): Unit = () + +} diff --git a/src/main/scala/org/monarchinitiative/dosdp/SPARQL.scala b/src/main/scala/org/monarchinitiative/dosdp/SPARQL.scala index 3d5411b..53a30e2 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/SPARQL.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/SPARQL.scala @@ -5,9 +5,10 @@ import java.util.regex.Pattern import org.apache.jena.query.ParameterizedSparqlString import org.monarchinitiative.dosdp.cli.Config.{AnnotationAxioms, AxiomKind, LogicalAxioms} import org.monarchinitiative.dosdp.cli.{DOSDPError, Generate} -import org.phenoscape.owlet.OwletManchesterSyntaxDataType.SerializableClassExpression +import org.monarchinitiative.dosdp.cli.DOSDPError.logError import org.semanticweb.owlapi.apibinding.OWLManager import org.semanticweb.owlapi.model._ +import org.semanticweb.owlapi.reasoner.OWLReasoner import org.phenoscape.scowl._ import zio.{Config => _, _} @@ -17,13 +18,13 @@ object SPARQL { private val factory = OWLManager.getOWLDataFactory() - def queryFor(compiled: CompiledPattern, axioms: AxiomKind): IO[DOSDPError, String] = { + def queryFor(compiled: CompiledPattern, axioms: AxiomKind, reasonerOpt: Option[OWLReasoner] = None): IO[DOSDPError, String] = { // Logical axioms always materialize even on an annotation-only query: the // `OPTIONAL { ?var rdfs:label ... }` clauses are derived from their // variable set. Compute once and share with `selectFor` / `triplesFor` so // the placeholder expansion runs only once per query. val logicalAxioms = Expansion.placeholderAxioms(compiled, LogicalAxioms) - triplesFor(compiled, axioms, logicalAxioms).map { triples => + triplesFor(compiled, axioms, logicalAxioms, reasonerOpt).map { triples => val select = selectFor(axioms, logicalAxioms) s""" PREFIX rdf: @@ -63,9 +64,9 @@ ORDER BY ?defined_class_label private val Thing = OWLManager.getOWLDataFactory.getOWLThing def triplesFor(compiled: CompiledPattern, axioms: AxiomKind): IO[DOSDPError, Seq[String]] = - triplesFor(compiled, axioms, Expansion.placeholderAxioms(compiled, LogicalAxioms)) + triplesFor(compiled, axioms, Expansion.placeholderAxioms(compiled, LogicalAxioms), None) - private def triplesFor(compiled: CompiledPattern, axioms: AxiomKind, logicalAxioms: Set[OWLAxiom]): IO[DOSDPError, Seq[String]] = { + private def triplesFor(compiled: CompiledPattern, axioms: AxiomKind, logicalAxioms: Set[OWLAxiom], reasonerOpt: Option[OWLReasoner]): IO[DOSDPError, Seq[String]] = { val props = compiled.readableIdentifierProperties.to(Set) val (queryLogical, queryAnnotations) = Generate.axiomsOutputChoice(axioms) val annotationAxioms = if (queryAnnotations) Expansion.placeholderAxioms(compiled, AnnotationAxioms) else Set.empty[OWLAxiom] @@ -73,26 +74,55 @@ ORDER BY ?defined_class_label annotationTriples <- ZIO.foreach(annotationAxioms.to(Seq))(triplesForAxiom(_, props)).map(_.flatten) axiomTriples <- if (queryLogical) ZIO.foreach(logicalAxioms.to(Seq))(triplesForAxiom(_, props)).map(_.flatten) else ZIO.succeed(Nil) varExpressions <- VarRangeExpressions.varExpressions(compiled) - variableTriples = varExpressions.toSeq.flatMap { + variableTriples <- ZIO.foreach(varExpressions.toSeq) { case (_, Thing) => - Seq.empty // relationships to owl:Thing are not typically explicit in the ontology + ZIO.succeed(Seq.empty[String]) // relationships to owl:Thing are not typically explicit in the ontology case (variable, named: OWLClass) => - Seq(s"?${DOSDP.processedVariable(variable)} rdfs:subClassOf* <${named.getIRI}> .") + ZIO.succeed(Seq(s"?${DOSDP.processedVariable(variable)} rdfs:subClassOf* <${named.getIRI}> .")) case (variable, ObjectUnionOf(operands)) if operands.forall(_.isNamed) => - Seq(operands.map(named => s"{ ?${DOSDP.processedVariable(variable)} rdfs:subClassOf* <${named.asOWLClass().getIRI}> . }") - .mkString(" UNION ")) + ZIO.succeed(Seq(operands.map(named => s"{ ?${DOSDP.processedVariable(variable)} rdfs:subClassOf* <${named.asOWLClass().getIRI}> . }") + .mkString(" UNION "))) case (variable, ObjectIntersectionOf(operands)) if operands.forall(_.isNamed) => - operands.map(named => s"?${DOSDP.processedVariable(variable)} rdfs:subClassOf* <${named.asOWLClass().getIRI}> . ") + ZIO.succeed(operands.map(named => s"?${DOSDP.processedVariable(variable)} rdfs:subClassOf* <${named.asOWLClass().getIRI}> . ").toSeq) case (variable, expression) => - val pss = new ParameterizedSparqlString() - pss.appendNode(expression.asOMN) - val sanitizedExpression = pss.toString - Seq(s"?${DOSDP.processedVariable(variable)} rdfs:subClassOf $sanitizedExpression .") - } + reasonerOpt match { + case Some(reasoner) => reasonerValues(variable, expression, reasoner) + case None => missingReasonerValues(variable, expression) + } + }.map(_.flatten) labelTriples = axiomVariables(logicalAxioms).map(v => s"OPTIONAL { $v rdfs:label ${v}__label . }") } yield annotationTriples ++ axiomTriples ++ variableTriples ++ labelTriples } + private def missingReasonerValues(variable: String, expression: OWLClassExpression): UIO[Seq[String]] = + ZIO.logWarning( + s"Variable range '$variable' uses anonymous class expression '${expression.toString}'. " + + "Query results for anonymous class expressions require requesting a reasoner." + ).as(Seq(s"VALUES ?${DOSDP.processedVariable(variable)} { }")) + + private def reasonerValues(variable: String, expression: OWLClassExpression, reasoner: OWLReasoner): IO[DOSDPError, Seq[String]] = + ZIO.attempt { + val classes = reasoner.synchronized(classesFor(reasoner, expression)) + .toSeq + .sortBy(_.getIRI.toString) + .map(cls => s"<${cls.getIRI}>") + .mkString(" ") + Seq(s"VALUES ?${DOSDP.processedVariable(variable)} { $classes }") + }.flatMapError(e => logError(s"Failed to expand variable range '$variable' with reasoner", e)) + + private def classesFor(reasoner: OWLReasoner, expression: OWLClassExpression): Set[OWLClass] = { + val subclasses = reasoner.getSubClasses(expression, false).getFlattened.asScala.toSet + // ELK 0.6.0 can omit direct subclasses from anonymous-expression queries when direct=false: + // https://github.com/liveontologies/elk-reasoner/issues/70 + val directSubclasses = reasoner.getSubClasses(expression, true).getFlattened.asScala.toSet + val equivalents = reasoner.getEquivalentClasses(expression).getEntities.asScala.toSet + val expressionIfNamed = expression match { + case named: OWLClass => Set(named) + case _ => Set.empty[OWLClass] + } + (subclasses ++ directSubclasses ++ equivalents ++ expressionIfNamed).filterNot(_.isOWLNothing) + } + def triplesForAxiom(axiom: OWLAxiom, readableIdentifierProperties: Set[OWLAnnotationProperty]): UIO[Seq[String]] = axiom match { case subClassOf: OWLSubClassOfAxiom => val (subClass, subClassTriples) = triplesForClassExpression(subClassOf.getSubClass) diff --git a/src/main/scala/org/monarchinitiative/dosdp/Utilities.scala b/src/main/scala/org/monarchinitiative/dosdp/Utilities.scala index 166019f..cad93fb 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/Utilities.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/Utilities.scala @@ -2,7 +2,6 @@ package org.monarchinitiative.dosdp import org.monarchinitiative.dosdp.cli.DOSDPError import org.monarchinitiative.dosdp.cli.DOSDPError.logError -import org.obolibrary.robot.CatalogXmlIRIMapper import org.semanticweb.owlapi.apibinding.OWLManager import org.semanticweb.owlapi.formats.FunctionalSyntaxDocumentFormat import org.semanticweb.owlapi.model.{IRI, OWLAxiom, OWLOntology} diff --git a/src/main/scala/org/monarchinitiative/dosdp/cli/Docs.scala b/src/main/scala/org/monarchinitiative/dosdp/cli/Docs.scala index 21da81f..043a073 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/cli/Docs.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/cli/Docs.scala @@ -1,12 +1,10 @@ package org.monarchinitiative.dosdp.cli import com.github.tototoshi.csv.CSVFormat -import org.geneontology.owl.differ.ManchesterSyntaxOWLObjectRenderer -import org.geneontology.owl.differ.shortform.MarkdownLinkShortFormProvider import org.monarchinitiative.dosdp.cli.DOSDPError.{logError, logErrorFail} import org.monarchinitiative.dosdp.cli.Generate.readFillers import org.monarchinitiative.dosdp.cli.Prototype.OboInOwlSource -import org.monarchinitiative.dosdp.{DOSDP, DocsMarkdown, PatternCompiler, Prefixes} +import org.monarchinitiative.dosdp.{DOSDP, DocsMarkdown, ManchesterSyntaxOWLObjectRenderer, MarkdownLinkShortFormProvider, PatternCompiler, Prefixes} import org.phenoscape.scowl._ import org.semanticweb.owlapi.apibinding.OWLManager import org.semanticweb.owlapi.io.OWLObjectRenderer diff --git a/src/main/scala/org/monarchinitiative/dosdp/cli/Prototype.scala b/src/main/scala/org/monarchinitiative/dosdp/cli/Prototype.scala index a216b34..fe868e8 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/cli/Prototype.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/cli/Prototype.scala @@ -1,6 +1,5 @@ package org.monarchinitiative.dosdp.cli -import better.files._ import org.eclipse.rdf4j.model.vocabulary.DCTERMS import org.monarchinitiative.dosdp.cli.DOSDPError.{logError, logErrorFail} import org.monarchinitiative.dosdp.{DOSDP, Utilities} @@ -8,6 +7,10 @@ import org.phenoscape.scowl._ import org.semanticweb.owlapi.model.{OWLAnnotationProperty, OWLAxiom} import zio.{Config => _, _} +import java.nio.file.{Files, Path, Paths} +import java.util.Locale +import scala.jdk.CollectionConverters._ + object Prototype { private val DCTTitle: OWLAnnotationProperty = AnnotationProperty(DCTERMS.TITLE.stringValue) @@ -15,14 +18,12 @@ object Prototype { def run(config: PrototypeConfig): IO[DOSDPError, Unit] = { Main.withLogContext(Map("command" -> "prototype")) { - val possibleFile = File(config.common.template) + val possibleFile = Paths.get(config.common.template) for { - isDir <- ZIO.attempt(possibleFile.isDirectory).flatMapError(e => logError(s"Unable to read input at $possibleFile", e)) + isDir <- ZIO.attempt(Files.isDirectory(possibleFile)).flatMapError(e => logError(s"Unable to read input at $possibleFile", e)) filenames <- if (isDir) { - ZIO.attempt { - possibleFile.list.filter { f => - f.extension(false, false, true).exists(e => (e == "yaml") || (e == "yml")) - }.map(_.toString).toSet + ZIO.attempt(Files.list(possibleFile)).acquireReleaseWithAuto { paths => + ZIO.attempt(paths.iterator.asScala.filter(isYamlFile).map(_.toString).toSet) }.flatMapError(e => logError(s"Couldn't list files in $possibleFile", e)) } else ZIO.succeed(Set(config.common.template)) dosdps <- ZIO.foreach(filenames)(f => Config.inputDOSDPFrom(f)) @@ -48,4 +49,9 @@ object Prototype { } yield axioms ++ maybeTitleAxiom } + private def isYamlFile(path: Path): Boolean = { + val name = path.getFileName.toString.toLowerCase(Locale.ROOT) + name.endsWith(".yaml") || name.endsWith(".yml") + } + } diff --git a/src/main/scala/org/monarchinitiative/dosdp/cli/Query.scala b/src/main/scala/org/monarchinitiative/dosdp/cli/Query.scala index ea3a255..90a3388 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/cli/Query.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/cli/Query.scala @@ -7,7 +7,6 @@ import org.apache.jena.vocabulary.DCTerms import org.monarchinitiative.dosdp.cli.Config.AxiomKind import org.monarchinitiative.dosdp.cli.DOSDPError.{logError, logErrorFail} import org.monarchinitiative.dosdp.{DOSDP, PatternCompiler, SPARQL, SesameJena} -import org.phenoscape.owlet.Owlet import org.phenoscape.scowl._ import org.semanticweb.HermiT.ReasonerFactory import org.semanticweb.elk.owlapi.ElkReasonerFactory @@ -96,10 +95,8 @@ object Query { def makeProcessedQuery(dosdp: DOSDP, prefixes: PartialFunction[String, String], axiomKind: AxiomKind, reasonerOpt: Option[OWLReasoner]): IO[DOSDPError, String] = for { compiled <- PatternCompiler.compile(dosdp, prefixes) - sparqlQuery <- SPARQL.queryFor(compiled, axiomKind) - } yield reasonerOpt - .map(reasoner => new Owlet(reasoner).expandQueryString(sparqlQuery, asValues = true)) - .getOrElse(sparqlQuery) + sparqlQuery <- SPARQL.queryFor(compiled, axiomKind, reasonerOpt) + } yield sparqlQuery private def processTarget(target: QueryTarget, config: QueryConfig, diff --git a/src/main/scala/org/monarchinitiative/dosdp/cli/Terms.scala b/src/main/scala/org/monarchinitiative/dosdp/cli/Terms.scala index 26381d4..ebc4239 100644 --- a/src/main/scala/org/monarchinitiative/dosdp/cli/Terms.scala +++ b/src/main/scala/org/monarchinitiative/dosdp/cli/Terms.scala @@ -1,12 +1,12 @@ package org.monarchinitiative.dosdp.cli -import better.files._ import com.github.tototoshi.csv.CSVReader import org.monarchinitiative.dosdp.cli.DOSDPError.logError import org.monarchinitiative.dosdp.{DOSDP, Expansion, PatternCompiler, Prefixes} import zio.{Config => _, _} import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths, StandardOpenOption} import scala.jdk.CollectionConverters._ object Terms { @@ -23,7 +23,16 @@ object Terms { .flatMapError(e => logError(s"Could not read fillers file at ${config.infile}", e)) identifiers = rows.flatMap(identifiersForRow(_, dosdp)).to(Set) iris = patternTerms ++ identifiers.flatMap(Prefixes.idToIRI(_, prefixes)) //FIXME should we report failure to expand to IRI? - _ <- ZIO.attempt(config.common.outfile.toFile.overwrite("").appendLines(iris.map(_.toString).toSeq: _*)(StandardCharsets.UTF_8)) + _ <- ZIO.attempt { + Files.write( + Paths.get(config.common.outfile), + iris.map(_.toString).toSeq.asJava, + StandardCharsets.UTF_8, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE + ) + }.unit .flatMapError(e => logError(s"Failed writing output file at ${config.common.outfile}", e)) } yield () } @@ -45,4 +54,3 @@ object Terms { } } - diff --git a/src/test/scala/org/monarchinitiative/dosdp/CatalogXmlIRIMapperTest.scala b/src/test/scala/org/monarchinitiative/dosdp/CatalogXmlIRIMapperTest.scala new file mode 100644 index 0000000..31cee02 --- /dev/null +++ b/src/test/scala/org/monarchinitiative/dosdp/CatalogXmlIRIMapperTest.scala @@ -0,0 +1,82 @@ +package org.monarchinitiative.dosdp + +import org.semanticweb.owlapi.model.IRI +import zio.ZIO +import zio.test._ + +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +object CatalogXmlIRIMapperTest extends ZIOSpecDefault { + + override def spec: Spec[TestEnvironment, Any] = + suite("CatalogXmlIRIMapper")( + test("resolves relative uri entries relative to the catalog file") { + for { + dir <- ZIO.attempt(Files.createTempDirectory("dosdp-catalog-")) + ontologyPath <- ZIO.attempt(Files.createFile(dir.resolve("imports.owl"))) + catalogPath = dir.resolve("catalog.xml") + catalog = + """ + | + | + | + |""".stripMargin + _ <- ZIO.attempt(Files.write(catalogPath, catalog.getBytes(StandardCharsets.UTF_8))) + mapper = new CatalogXmlIRIMapper(catalogPath.toFile) + mapped = mapper.getDocumentIRI(IRI.create("http://example.org/imports.owl")) + expected = IRI.create(ontologyPath.toFile.getCanonicalFile) + } yield assertTrue(mapped == expected) + }, + test("keeps absolute IRI mappings as IRIs") { + for { + catalogPath <- ZIO.attempt(Files.createTempFile("dosdp-catalog-", ".xml")) + catalog = + """ + | + | + | + |""".stripMargin + _ <- ZIO.attempt(Files.write(catalogPath, catalog.getBytes(StandardCharsets.UTF_8))) + mapper = new CatalogXmlIRIMapper(catalogPath.toFile) + mapped = mapper.getDocumentIRI(IRI.create("http://example.org/ontology.owl")) + } yield assertTrue(mapped == IRI.create("https://example.org/mirror/ontology.owl")) + }, + test("accepts catalog files with an external DTD declaration") { + for { + dir <- ZIO.attempt(Files.createTempDirectory("dosdp-catalog-")) + ontologyPath <- ZIO.attempt(Files.createFile(dir.resolve("imports.owl"))) + catalogPath = dir.resolve("catalog.xml") + catalog = + """ + | + | + | + | + |""".stripMargin + _ <- ZIO.attempt(Files.write(catalogPath, catalog.getBytes(StandardCharsets.UTF_8))) + mapper = new CatalogXmlIRIMapper(catalogPath.toFile) + mapped = mapper.getDocumentIRI(IRI.create("http://example.org/imports.owl")) + expected = IRI.create(ontologyPath.toFile.getCanonicalFile) + } yield assertTrue(mapped == expected) + }, + test("accepts prefixed catalog uri entries") { + for { + dir <- ZIO.attempt(Files.createTempDirectory("dosdp-catalog-")) + ontologyPath <- ZIO.attempt(Files.createFile(dir.resolve("imports.owl"))) + catalogPath = dir.resolve("catalog.xml") + catalog = + """ + | + | + | + |""".stripMargin + _ <- ZIO.attempt(Files.write(catalogPath, catalog.getBytes(StandardCharsets.UTF_8))) + mapper = new CatalogXmlIRIMapper(catalogPath.toFile) + mapped = mapper.getDocumentIRI(IRI.create("http://example.org/imports.owl")) + expected = IRI.create(ontologyPath.toFile.getCanonicalFile) + } yield assertTrue(mapped == expected) + } + ) + +} diff --git a/src/test/scala/org/monarchinitiative/dosdp/GenerateDefinedClassTest.scala b/src/test/scala/org/monarchinitiative/dosdp/GenerateDefinedClassTest.scala index 456810e..a67602e 100644 --- a/src/test/scala/org/monarchinitiative/dosdp/GenerateDefinedClassTest.scala +++ b/src/test/scala/org/monarchinitiative/dosdp/GenerateDefinedClassTest.scala @@ -8,10 +8,10 @@ import zio.test._ // Pins the --generate-defined-class IRI-minting path: when generateDefinedClass = true // and the row has no `defined_class` column, the defined-class IRI is computed as -// sha1Hex(sorted-bindings) appended to the pattern IRI as a fragment. A refactor of -// DOSDP.computeDefinedIRI or of the binding-collection that feeds it must keep the -// same IRI shape, or every consumer that relies on these IRIs to dedupe rows breaks -// silently. +// lowercase SHA-1 hex of the sorted bindings appended to the pattern IRI as a +// fragment. A refactor of DOSDP.computeDefinedIRI or of the binding-collection +// that feeds it must keep the same IRI shape, or every consumer that relies on +// these IRIs to dedupe rows breaks silently. object GenerateDefinedClassTest extends ZIOSpecDefault { private val patternIRIString = "http://purl.obolibrary.org/obo/test/generate_defined_class_test.yaml" @@ -28,10 +28,12 @@ object GenerateDefinedClassTest extends ZIOSpecDefault { name = Some(PrintfAnnotationOBO(None, None, Some("%s item"), Some(List("item")), None)), subClassOf = Some(PrintfOWLConvenience(None, Some("'part_of' some %s"), Some(List("item"))))) - // Mirror what Generate.renderPattern computes internally, so that the assertion - // would catch a behavior change in DOSDP.computeDefinedIRI itself. + // Mirror what Generate.renderPattern computes internally for the rendering assertions, + // and pin the literal IRI string below to catch digest-format regressions. private val expectedDefinedIRI: IRI = DOSDP.computeDefinedIRI(patternIRI, Map("item" -> SingleValue("ONT:0000002"))) + private val expectedDefinedIRIString: String = + s"$patternIRIString#b241301b4391f87c756c868293230e2464e77d44" private val expectedDefinedClass: OWLClass = Class(expectedDefinedIRI) private val expectedLabel: OWLAnnotationAssertionAxiom = expectedDefinedClass Annotation(RDFSLabel, "http://purl.obolibrary.org/obo/ONT_0000002 item") @@ -39,6 +41,9 @@ object GenerateDefinedClassTest extends ZIOSpecDefault { expectedDefinedClass SubClassOf (partOf some item) def spec = suite("generate-defined-class")( + test("computeDefinedIRI uses lowercase SHA-1 hex") { + assert(expectedDefinedIRI.toString)(equalTo(expectedDefinedIRIString)) + }, test("mints defined-class IRI from sha1 of bindings when generateDefinedClass = true") { val row = Map("item" -> "ONT:0000002") for { @@ -67,4 +72,4 @@ object GenerateDefinedClassTest extends ZIOSpecDefault { } ) -} \ No newline at end of file +} diff --git a/src/test/scala/org/monarchinitiative/dosdp/MarkdownRenderingTest.scala b/src/test/scala/org/monarchinitiative/dosdp/MarkdownRenderingTest.scala new file mode 100644 index 0000000..6af5352 --- /dev/null +++ b/src/test/scala/org/monarchinitiative/dosdp/MarkdownRenderingTest.scala @@ -0,0 +1,40 @@ +package org.monarchinitiative.dosdp + +import org.semanticweb.owlapi.apibinding.OWLManager +import org.semanticweb.owlapi.model.{IRI, OWLEntity} +import org.semanticweb.owlapi.util.ShortFormProvider +import zio.test._ + +object MarkdownRenderingTest extends ZIOSpecDefault { + + private val factory = OWLManager.getOWLDataFactory + + private def shortFormProvider(shortForm: String): ShortFormProvider = + new ShortFormProvider { + override def getShortForm(entity: OWLEntity): String = shortForm + + override def dispose(): Unit = () + } + + def spec = suite("Markdown rendering")( + test("Markdown short forms wrap labels in links") { + val entity = factory.getOWLClass(IRI.create("http://example.org/Thing")) + val provider = new MarkdownLinkShortFormProvider(shortFormProvider("Example thing")) + + assert(provider.getShortForm(entity))(Assertion.equalTo("[Example thing](http://example.org/Thing)")) + }, + test("Manchester renderer applies short forms to bare IRIs") { + val renderer = new ManchesterSyntaxOWLObjectRenderer() + renderer.setShortFormProvider(shortFormProvider("Example thing")) + + assert(renderer.render(IRI.create("http://example.org/Thing")))(Assertion.equalTo("Example thing")) + }, + test("Manchester renderer HTML-escapes string literals") { + val renderer = new ManchesterSyntaxOWLObjectRenderer() + val literal = factory.getOWLLiteral("alpha < beta & \"gamma\"") + + assert(renderer.render(literal))(Assertion.equalTo("\"alpha < beta & "gamma"\"^^string")) + } + ) + +} diff --git a/src/test/scala/org/monarchinitiative/dosdp/QueryReasonerTest.scala b/src/test/scala/org/monarchinitiative/dosdp/QueryReasonerTest.scala new file mode 100644 index 0000000..5bbb919 --- /dev/null +++ b/src/test/scala/org/monarchinitiative/dosdp/QueryReasonerTest.scala @@ -0,0 +1,80 @@ +package org.monarchinitiative.dosdp + +import org.monarchinitiative.dosdp.cli.{Config, Query} +import org.semanticweb.elk.owlapi.ElkReasonerFactory +import org.semanticweb.owlapi.apibinding.OWLManager +import org.semanticweb.owlapi.model.IRI +import org.semanticweb.owlapi.reasoner.InferenceType +import zio.{Config => _, _} +import zio.test.Assertion._ +import zio.test._ + +import scala.jdk.CollectionConverters._ + +object QueryReasonerTest extends ZIOSpecDefault { + + private val prefixes: Map[String, String] = Map( + "BFO" -> "http://purl.obolibrary.org/obo/BFO_", + "EX" -> "http://example.org/" + ) + + private val pattern = DOSDP( + classes = Some(Map("target" -> "EX:Target")), + relations = Some(Map("part_of" -> "BFO:0000050")), + vars = Some(Map("entity" -> "'part_of' some 'target'")), + subClassOf = Some(PrintfOWLConvenience( + annotations = None, + text = Some("%s"), + vars = Some(List("entity")) + )) + ) + + def spec = suite("Query reasoner expansion")( + test("anonymous variable ranges without a reasoner produce empty VALUES") { + for { + query <- Query.makeProcessedQuery(pattern, prefixes, Config.LogicalAxioms, None) + } yield assert(query)(containsString("VALUES ?entity { }")) && + assert(query)(not(containsString("'part_of' some 'target'"))) + }, + test("ELK expands complex variable ranges into direct VALUES blocks") { + ZIO.scoped { + for { + ontology <- ZIO.attempt { + val manager = OWLManager.createOWLOntologyManager() + val factory = manager.getOWLDataFactory + val target = factory.getOWLClass(IRI.create("http://example.org/Target")) + val partOfTarget = factory.getOWLClass(IRI.create("http://example.org/PartOfTarget")) + val candidate = factory.getOWLClass(IRI.create("http://example.org/Candidate")) + val directCandidate = factory.getOWLClass(IRI.create("http://example.org/DirectCandidate")) + val nonMatching = factory.getOWLClass(IRI.create("http://example.org/NonMatching")) + val partOf = factory.getOWLObjectProperty(IRI.create("http://purl.obolibrary.org/obo/BFO_0000050")) + val ontology = manager.createOntology() + manager.addAxiom(ontology, factory.getOWLDeclarationAxiom(target)) + manager.addAxiom(ontology, factory.getOWLDeclarationAxiom(partOfTarget)) + manager.addAxiom(ontology, factory.getOWLDeclarationAxiom(candidate)) + manager.addAxiom(ontology, factory.getOWLDeclarationAxiom(directCandidate)) + manager.addAxiom(ontology, factory.getOWLDeclarationAxiom(partOf)) + manager.addAxiom(ontology, factory.getOWLEquivalentClassesAxiom(partOfTarget, factory.getOWLObjectSomeValuesFrom(partOf, target))) + manager.addAxiom(ontology, factory.getOWLSubClassOfAxiom(candidate, partOfTarget)) + manager.addAxiom(ontology, factory.getOWLSubClassOfAxiom(directCandidate, factory.getOWLObjectSomeValuesFrom(partOf, target))) + manager.addAxiom(ontology, factory.getOWLDeclarationAxiom(nonMatching)) + ontology + } + reasoner <- ZIO.acquireRelease( + ZIO.attempt(new ElkReasonerFactory().createReasoner(ontology)) + )(reasoner => ZIO.succeed(reasoner.dispose())) + _ <- ZIO.attempt(reasoner.precomputeInferences(InferenceType.CLASS_HIERARCHY)) + query <- Query.makeProcessedQuery(pattern, prefixes, Config.LogicalAxioms, Some(reasoner)) + queryClasses = ontology.getClassesInSignature().asScala.exists(_.getIRI.toString.startsWith("urn:dosdp:query:")) + } yield assert(query)(containsString("VALUES")) && + assert(query)(containsString("?entity")) && + assert(query)(containsString("http://example.org/PartOfTarget")) && + assert(query)(containsString("http://example.org/DirectCandidate")) && + assert(query)(not(containsString("http://example.org/NonMatching"))) && + assert(query)(not(containsString("urn:dosdp:query:"))) && + assert(queryClasses)(isFalse) + } + } + ) + +}