diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 00000000..bf8fceb6 --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,88 @@ +name: Publish python distribution to PyPI and TestPyPI + +on: + workflow_dispatch: + inputs: + pushTestPyPi: + description: 'Push package to TestPyPi' + required: true + type: boolean + pushPyPi: + description: 'Push package to PyPi' + required: true + type: boolean + +jobs: + build-pypi: + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux_2_28_x86_64 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # We need full history for setuptools_scm to figure out version + - name: Set safe directory (work around checkout not doing that properly for containers) + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Install build dependencies for checktestdata + run: yum -y install boost-devel gmp-devel + - name: Build sdist (and broken wheel) + run: /opt/python/cp311-cp311/bin/python -m build + - name: Repair wheel + run: auditwheel repair dist/problemtools-*.whl + - name: Replace broken wheel with repaired wheel + run: | + rm -f dist/*.whl + cp wheelhouse/*.whl dist + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-testpypi: + name: Publish Python distribution to TestPyPI + needs: + - build-pypi + runs-on: ubuntu-latest + if: ${{ inputs.pushTestPyPi }} + + environment: + name: testpypi + url: https://test.pypi.org/p/problemtools + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + publish-to-pypi: + name: Publish Python distribution to PyPI + needs: + - build-pypi + runs-on: ubuntu-latest + if: ${{ inputs.pushPyPi }} + + environment: + name: pypi + url: https://pypi.org/p/problemtools + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 00000000..b3226dfe --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,62 @@ +# This workflow will install Python dependencies, run tests and lint +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python tests + +on: + push: + branches: [ "master", "develop" ] + pull_request: + branches: [ "master", "develop" ] + +permissions: + contents: read + +jobs: + pythontests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] # 3.11 is the lowest we support, since we want StrEnum + container: + image: problemtools/githubci:latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m venv venv + venv/bin/python --version + venv/bin/pip install mypy ruff pytest + if [ -f requirements.txt ]; then venv/bin/pip install -r requirements.txt; fi + - name: Lint with ruff + run: venv/bin/ruff check --output-format=github + - name: Check ruff formatting + run: venv/bin/ruff format --check --diff + - name: Test with pytest + run: venv/bin/pytest + - name: Run mypy + run: | + venv/bin/mypy --non-interactive --config-file mypy.ini -p problemtools + + packages: # Use a separate job to test debian packaging to speed things up (no need to test this for every python version above) + runs-on: ubuntu-latest + container: + image: problemtools/githubci:latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build debian packages + run: | + make builddeb + - name: Install debian package + run: dpkg -i ../kattis-problemtools_*.deb + - name: Verify examples + run: | + shopt -s extglob + verifyproblem examples/!(README.md) + shell: bash diff --git a/.gitignore b/.gitignore index ddbe0378..18a3eccb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ *.pyc *~ +*.swp /.cache/ /problemtools.egg-info/ /support/default_validator/default_validator /support/interactive/interactive +build/ +/problemtools/_version.py + +venv/ +.pytest_cache/ +.mypy_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..889c8316 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.8 + hooks: + - id: ruff + - id: ruff-format diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f81f3fcd..00000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -language: python -python: - - 3.7 - -script: py.test diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e9787418..00000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM ubuntu:20.04 - -MAINTAINER austrin@kattis.com - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y \ - automake \ - g++ \ - git \ - libboost-all-dev \ - libgmp-dev \ - libgmp10 \ - libgmpxx4ldbl \ - openjdk-8-jdk \ - python3-minimal \ - python3-pip \ - python3-plastex \ - python3-yaml \ - sudo \ - texlive-fonts-recommended \ - texlive-lang-cyrillic \ - texlive-latex-extra \ - texlive-plain-generic \ - tidy \ - vim - -RUN pip3 install git+https://github.com/kattis/problemtools diff --git a/LICENSE b/LICENSE index 84c39ef9..01cc176a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2010-2019 Kattis and all respective contributors +Copyright (c) Kattis and all respective contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index 4d57d2ed..06421ee4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,9 @@ recursive-include problemtools/config * recursive-include problemtools/templates * recursive-include problemtools/tests * +recursive-include examples * recursive-include support * +recursive-include tests * +global-exclude */__pycache__/* +global-exclude *.pyc +recursive-exclude .github * diff --git a/Makefile b/Makefile index 296634e5..bd3337bf 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,7 @@ checktestdata: support/checktestdata/bootstrap support/checktestdata/bootstrap: git submodule update --init + +clean: + make -C support clean + rm -rf problemtools.egg-info build diff --git a/README.md b/README.md index 1bba9f93..98e51c2d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Kattis Problem Tools -master: -[![Master Build Status](https://travis-ci.org/Kattis/problemtools.svg?branch=master)](https://travis-ci.org/Kattis/problemtools). -develop: -[![Develop Build Status](https://travis-ci.org/Kattis/problemtools.svg?branch=develop)](https://travis-ci.org/Kattis/problemtools) +![Build Status](https://github.com/kattis/problemtools/actions/workflows/python-app.yml/badge.svg?branch=master) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) -These are tools to manage problem packages using the Kattis problem package -format. +These are tools to manage problem packages using the Kattis [problem package +format](https://www.kattis.com/problem-package-format/). The problem package +specification is developed in the [problem package format +repository](https://github.com/Kattis/problem-package-format). ## Programs Provided @@ -21,6 +21,22 @@ Running any of them with the command-line option `-h` gives documentation on what arguments they accept. +## Format versions + +There are currently two versions of the problem package format, +[legacy](https://www.kattis.com/problem-package-format/spec/legacy.html) and +[2023-07-draft](https://www.kattis.com/problem-package-format/spec/2023-07-draft.html). +We have begun work on supporting 2023-07-draft, but there are *many* changes +between legacy and 2023-07-draft which are not yet implemented. Verifyproblem +will do its best to parse and verify problems in 2023-07-draft, but key things +like scoring still behave like legacy. Also, note that 2023-07 is still a draft +standard. + +We advice against packaging production problems in 2023-07-draft. Especially so +if you plan to have problems installed on [Kattis](https://open.kattis.com), where +we currently *only* support installing legacy problems. + + ## Example Problems A few examples of problem packages can be found in [examples](examples). @@ -31,20 +47,21 @@ A few examples of problem packages can be found in [examples](examples). There are four supported ways of installing and running problemtools. (For non-Linux users, "Method 2" below, to use Docker, is probably the least painful.) -### Method 1: Install the Python package +Note that in all methods except for "Method 2", you must manually install +dependencies such as LaTeX and tools for any languages you want to use. See +[Requirements and compatbility](#requirements-and-compatibility) for details. + +### Method 1: Install the Python package using pipx Run ``` -sudo pip3 install git+https://github.com/kattis/problemtools +pipx install problemtools ``` -Or if you don't want a system-wide installation, -``` -pip3 install --user git+https://github.com/kattis/problemtools -``` -With this second option, in order to get the command line scripts, you need -to make sure that the local user bin path used (e.g., on Linux, -`$HOME/.local/bin`) is in your `$PATH`. +In order to get the command line scripts, you need to make sure that the local +user bin path used (e.g., on Linux, `$HOME/.local/bin`) is in your `$PATH`. See +[pipx' installation instructions](https://pipx.pypa.io/stable/installation/) +for information on how to install `pipx` and set up your `$PATH`. In order for problemtools to build and run properly, you also need to have LaTeX and various LaTeX packages installed. See [Requirements and @@ -64,19 +81,30 @@ We maintain three official problemtools Docker images on Docker Hub: - [`problemtools/minimal`](https://hub.docker.com/r/problemtools/minimal/): this image only contains problemtools, no additional programming languages. As such, it is not particularly useful on its own, but if you are organizing a contest and want to set up a problemtools environment containing exactly the right set of compilers/interpreters for your contest, this is the recommended starting point. -For example, suppose you want to use the `problemtools/icpc` image. To get started, install the [Docker CLI](https://docs.docker.com/install), and then pull the image: +For example, suppose you want to use the `problemtools/icpc` image. To get started (or update to the latest release), install the [Docker CLI](https://docs.docker.com/install), and then pull the image: docker pull problemtools/icpc -Once the image has finished downloading, you can check that it exists on your system using `docker images`. To launch an interactive container and play around with *verifyproblem*, *problem2pdf*, and *problem2html* run: +The most convenient way to use the container is by creating shell script(s) similar to this and add it to your `$PATH`. If you call the script `verifyproblem.sh`, you could then `verifyproblem.sh examples/hello` to use the icpc docker image to verify examples/hello: +```sh +#!/bin/bash + +if [ $1 -a -d $1 ]; then + docker run --rm -t -v $(dirname $(readlink -f $1)):/work problemtools/icpc verifyproblem /work/$(basename $1) +else + echo No such directory: $1 +fi +``` + +To instead launch an interactive container and play around with *verifyproblem*, *problem2pdf*, and *problem2html* run: docker run --rm -it problemtools/icpc By default, docker containers do _NOT_ persist storage between runs, so any files you create or modify will be lost when the container stops running. Two common ways of dealing with this are: -1) Use a [bind mount](https://docs.docker.com/storage/bind-mounts/) to mount a directory on your machine into the docker container. This can be done as follows (see Docker documentation for further details): +1) Use a [bind mount](https://docs.docker.com/storage/bind-mounts/) to mount a directory on your machine into the docker container. Mounting the current directory to /kattis_work_dir can be done as follows (see Docker documentation for further details): ``` - docker run --rm -it -v ${FULL_PATH_TO_MOUNT}:/kattis_work_dir problemtools/icpc + docker run --rm -it -v $(pwd):/kattis_work_dir problemtools/icpc ``` 2) Persist any changes you want to keep to a remote file system/source control (e.g., a remote Git repository; note, however, that you would first need to install Git in the image). @@ -95,18 +123,26 @@ Hub, these are only updated sporadically for testing purposes and not kept up to date). -### Method 3: Run directly from the repository. +### Method 3: Run directly from the repository -If you intend to help develop problemtools, or if you just want a -bare-bones way of running them, this is your option. +If you intend to help develop problemtools, or if you just want a bare-bones +way of running them, this is your option. Note that python 3.11+ is required to +run problemtools (mentioned here, as this method of running problemtools will +not check python version for you). For this method, you need to clone the repository (just downloading a zip archive of it does not work because the project has submodules that are not included in that zip archive). -In order for the tools to work, you first have to compile the various -support programs, which can be done by running `make` in the root -directory of problemtools. +Set up your venv, install the requirements, and compile the various support +programs: + +```sh +cd ~/problemtools # Assuming you checked out problemtools in your home directory +python3 -m venv venv +venv/bin/pip install -r requirements.txt +make +``` When this is done, you can run the three programs `bin/verifyproblem.sh`, `bin/problem2pdf.sh`, and @@ -134,7 +170,7 @@ root of the repository). Apart from the build dependencies listed [below](#ubuntu), building the Debian package requires that the following tools are installed: - debhelper dh-python dpkg-dev + debhelper dh-virtualenv dpkg-dev The package can then be installed using (replace `` as appropriate): @@ -178,7 +214,7 @@ problemtools' configuration: are not sure whether you should use it, then you probably shouldn't. This file can be used to specify the system defaults for those problem limits which are not given a fixed default value in the - [problem format specification](http://www.problemarchive.org/wiki/index.php/Problem_Format#limits). + [problem format specification](https://www.kattis.com/problem-package-format/spec/2023-07-draft.html#limits). The system defaults assumed by problemtools can be found in (problemtools/config/problem.yaml). For instance, if you are primarily working against a system with a default memory limit of 2 GiB, @@ -189,35 +225,36 @@ problemtools' configuration: memory: 2048 # (unit is MiB) ``` - (In principle, it is possible to override the defaults of other values than the - system-dependent defaults in the problem.yaml metadata files this way, but such - usage is very strongly discouraged.) - ## Requirements and compatibility -To build and run the tools, you need Python 3 with the YAML and PlasTeX libraries, -and a LaTeX installation. +To build and run the tools, you need Python 3.11+ with the YAML and PlasTeX libraries, +and a LaTeX installation. You must also install language tools (e.g., compilers) +for any languages used in problem packages. ### Ubuntu The dependencies needed to *build/install* problemtools can be installed with: - sudo apt install automake g++ make libboost-regex-dev libgmp-dev libgmp10 libgmpxx4ldbl python3 python3-pytest python3-setuptools python3-yaml python3-plastex + sudo apt install python3-venv automake g++ make libboost-regex-dev libgmp-dev python3 git And the dependencies needed to *run* problemtools can be installed with: - sudo apt install ghostscript libgmpxx4ldbl python3-minimal python-pkg-resources python3-plastex python3-yaml texlive-fonts-recommended texlive-lang-cyrillic texlive-latex-extra texlive-plain-generic tidy + sudo apt install ghostscript pandoc python3 texlive-fonts-recommended texlive-latex-extra texlive-luatex texlive-plain-generic tidy dvisvgm + +To render problem statements to pdf with non-latin characters, also add the following fonts: + + sudo apt install fonts-cmu fonts-noto-color-emoji ### Fedora On Fedora, these dependencies can be installed with: - sudo dnf install boost-regex gcc gmp-devel gmp-c++ python3 python3-pyyaml texlive-latex texlive-collection-fontsrecommended texlive-fancyhdr texlive-subfigure texlive-wrapfig texlive-import texlive-ulem texlive-xifthen texlive-overpic texlive-pbox tidy ghostscript + sudo dnf install boost-regex gcc gmp-devel gmp-c++ pandoc python3 python3-pyyaml texlive-latex texlive-collection-fontsrecommended texlive-fancyhdr texlive-subfigure texlive-wrapfig texlive-import texlive-ulem texlive-xifthen texlive-overpic texlive-pbox tidy ghostscript Followed by: - pip3 install --user plastex + pip3 install --user plastex nh3 ### Arch Package is available on the AUR [kattis-problemtools-git](https://aur.archlinux.org/packages/kattis-problemtools-git). Use your favorite AUR helper or follow the installation instructions found [here](https://wiki.archlinux.org/title/Arch_User_Repository#Installing_and_upgrading_packages). diff --git a/admin/.gitignore b/admin/.gitignore new file mode 100644 index 00000000..a4d089c1 --- /dev/null +++ b/admin/.gitignore @@ -0,0 +1 @@ +pypi_dist/ diff --git a/admin/build_pypi_packages.sh b/admin/build_pypi_packages.sh new file mode 100755 index 00000000..d9567a24 --- /dev/null +++ b/admin/build_pypi_packages.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -e + +ALLOW_DIRTY=false +TAG=develop + +echo N.B., this script is solely to allow for local testing of whl/src. +echo To actually build and push things to pypi, trigger the pypi flow +echo in github actions, https://github.com/Kattis/problemtools/actions + +while getopts "d" opt; do + case $opt in + d) ALLOW_DIRTY=true ;; + \?) echo "Invalid option: -$opt" ;; + esac +done + +shift $((OPTIND-1)) + +if [ "$1" != "" ]; then + TAG=$1 +fi + +cd $(dirname $(readlink -f $0)) + +if ! ../venv/bin/twine -h > /dev/null 2> /dev/null; then + echo "Did not find twine. Please run ../venv/bin/pip install twine" + exit 1 +fi + +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +if [[ $(git rev-parse --abbrev-ref HEAD) != ${TAG} && $(git describe --exact-match --tags 2>/dev/null) != ${TAG} ]]; then + echo "Repository is currently not on branch/tag ${TAG}." + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +echo "Building sdist and manylinux wheel" +sudo rm -rf ./pypi_dist +docker run --rm -v $(pwd)/..:/problemtools -v $(pwd)/pypi_dist:/dist quay.io/pypa/manylinux_2_28_x86_64 /bin/bash -c " + yum -y install boost-devel gmp-devel ; + mkdir /build ; + cd /build ; + git config --global --add safe.directory /problemtools/.git ; + git clone /problemtools ; + cd problemtools ; + git checkout ${TAG} ; + /opt/python/cp311-cp311/bin/python -m build ; + auditwheel repair dist/problemtools-*.whl ; + cp dist/*.tar.gz /dist ; + cp wheelhouse/*.whl /dist" +sudo chown -R $USER:$USER pypi_dist + +../venv/bin/twine check pypi_dist/* + +echo "Running verifyproblem from wheel on all examples" +TEMPDIR=$(mktemp -d) +python3 -m venv "${TEMPDIR}" +"${TEMPDIR}/bin/pip" install pypi_dist/problemtools*manylinux*whl +shopt -s extglob +if ! "${TEMPDIR}/bin/verifyproblem" ../examples/!(README.md); then + echo "Running verifyproblem on all examples failed. Please review output above to debug." + rm -rf "${TEMPDIR}" + exit 1 +fi +rm -rf "${TEMPDIR}" + +echo "Sucessfully built packages. If you're happy with them, upload:" +echo " ../venv/bin/twine upload --verbose pypi_dist/*" diff --git a/admin/docker/Dockerfile.build b/admin/docker/Dockerfile.build index e2f7a3bf..b8080fe1 100644 --- a/admin/docker/Dockerfile.build +++ b/admin/docker/Dockerfile.build @@ -1,43 +1,24 @@ -# Package for building the problemtools .deb package -# Ends up in the /usr/local/problemtools_build/deb/ directory +# Docker image with all packages needed to build a problemtools .deb # -# Setting build argument PROBLEMTOOLS_VERSION causes a specific -# version of problemtools to be built (default is latest version of -# develop branch on GitHub) +# Not uploaded anywhere, only used locally during building -FROM ubuntu:22.04 - -LABEL maintainer="austrin@kattis.com" +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -# Install packages needed for build -RUN apt update && \ - apt install -y \ +# Packages required to build and run problemtools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ automake \ + build-essential \ debhelper \ - dh-python \ + dh-virtualenv \ dpkg-dev \ g++ \ git \ make \ - libboost-regex-dev \ - libgmp-dev \ - libgmp10 \ - libgmpxx4ldbl \ - python3 \ - python3-pytest \ - python3-setuptools \ - python3-yaml \ - python3-setuptools - -RUN mkdir -p /usr/local/problemtools_build - -WORKDIR /usr/local/problemtools_build -RUN git clone --recursive https://github.com/kattis/problemtools - -ARG PROBLEMTOOLS_VERSION=develop -RUN cd problemtools && git checkout ${PROBLEMTOOLS_VERSION} && make builddeb - -RUN mkdir -p deb -RUN mv kattis-problemtools*.deb deb/ + libboost-regex-dev diff --git a/admin/docker/Dockerfile.full b/admin/docker/Dockerfile.full index 40580dd6..75ec8745 100644 --- a/admin/docker/Dockerfile.full +++ b/admin/docker/Dockerfile.full @@ -1,32 +1,23 @@ # Full problemtools docker image, containing problemtools and all # supported programming languages. # +# +# Build requirements: +# - The problemtools .deb package must be available from the host file +# system under a file name matching +# artifacts/deb/kattis-problemtools*.deb +# (Version of that .deb file should match the build argument +# PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM problemtools/icpc:${PROBLEMTOOLS_VERSION} - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/fulllangs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y \ - fp-compiler \ - gfortran \ - gnucobol \ - gccgo \ - ghc haskell-platform \ - gnustep-devel gnustep gnustep-make gnustep-common gobjc \ - libgmp3-dev \ - libmozjs-78-dev \ - lua5.4 \ - mono-complete \ - nodejs \ - ocaml-nox \ - php-cli \ - pypy \ - rustc \ - sbcl \ - scala \ - swi-prolog \ - ; +RUN mkdir -p /usr/local/artifacts +WORKDIR /usr/local/artifacts +COPY artifacts/deb . +RUN dpkg -i kattis-problemtools*.deb + +WORKDIR / diff --git a/admin/docker/Dockerfile.fulllangs b/admin/docker/Dockerfile.fulllangs new file mode 100644 index 00000000..cc0dd3df --- /dev/null +++ b/admin/docker/Dockerfile.fulllangs @@ -0,0 +1,40 @@ +# Docker image with all packages needed to run a problemtools .deb, plus +# language support for all supported languages +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/icpclangs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# All languages, plus curl which we need to fetch pypy2 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + curl \ + fp-compiler \ + gfortran \ + gnucobol \ + gccgo \ + ghc \ + gnustep-devel gnustep gnustep-make gnustep-common gobjc \ + lua5.4 \ + mono-complete \ + nodejs \ + ocaml-nox \ + php-cli \ + rustc \ + sbcl \ + scala \ + swi-prolog + +# pypy2 is no longer packaged for Ubuntu, so download tarball (and check a sha256) +RUN curl -LO https://downloads.python.org/pypy/pypy2.7-v7.3.16-linux64.tar.bz2 \ + && echo '04b2fceb712d6f811274825b8a471ee392d3d1b53afc83eb3f42439ce00d8e07 pypy2.7-v7.3.16-linux64.tar.bz2' | sha256sum --check \ + && tar -xf pypy2.7-v7.3.16-linux64.tar.bz2 \ + && mv pypy2.7-v7.3.16-linux64 /opt/pypy \ + && ln -s /opt/pypy/bin/pypy /usr/bin/pypy \ + && rm pypy2.7-v7.3.16-linux64.tar.bz2 diff --git a/admin/docker/Dockerfile.githubci b/admin/docker/Dockerfile.githubci new file mode 100644 index 00000000..7c4b5217 --- /dev/null +++ b/admin/docker/Dockerfile.githubci @@ -0,0 +1,23 @@ +# Docker image with all deb packages needed for our github actions +# - Building a problemtools deb +# - Running verifyproblem on all examples + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/fulllangs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# Packages required to build and run problemtools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + automake \ + build-essential \ + debhelper \ + dh-virtualenv \ + dpkg-dev \ + git \ + make \ + libboost-regex-dev diff --git a/admin/docker/Dockerfile.icpc b/admin/docker/Dockerfile.icpc index 55f2dff6..6366fe01 100644 --- a/admin/docker/Dockerfile.icpc +++ b/admin/docker/Dockerfile.icpc @@ -1,48 +1,22 @@ # Basic problemtools docker image, containing problemtools and the # "ICPC languages" (C, C++, Java, Kotlin, and Python 3) # +# Build requirements: +# - The problemtools .deb package must be available from the host file +# system under a file name matching +# artifacts/deb/kattis-problemtools*.deb +# (Version of that .deb file should match the build argument +# PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM problemtools/minimal:${PROBLEMTOOLS_VERSION} - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/icpclangs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -# Install C++, Java, and PyPy 3 via their ppa repository (Kotlin is installed below to get a more up-to-date version) -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - add-apt-repository ppa:pypy/ppa && \ - apt-get update && \ - apt-get install -y \ - gcc g++ \ - openjdk-11-jdk openjdk-11-jre \ - pypy3 - -RUN apt-get install -y curl - -ARG KOTLIN_VERSION=1.8.10 -RUN curl "https://github.com/JetBrains/kotlin/releases/download/v$KOTLIN_VERSION/kotlin-compiler-$KOTLIN_VERSION.zip" -L -o kotlin.zip -RUN unzip -q kotlin.zip -RUN rm kotlin.zip - -RUN mv kotlinc/bin/* /usr/bin -RUN mv kotlinc/lib/* /usr/lib - -RUN rm -r kotlinc - -# Reconfigure problemtools: -# - Use PyPy for Python 2 (not available in this image but in the full one) -# - Use PyPy for Python 3 -RUN mkdir -p /etc/kattis/problemtools -RUN echo " \n\ -python2: \n\ - name: 'Python 2 w/PyPy'\n\ - run: '/usr/bin/pypy \"{mainfile}\"'\n\ - \n\ -python3: \n\ - name: 'Python 3 w/PyPy'\n\ - run: '/usr/bin/pypy3 \"{mainfile}\"'\n\ - \n" > /etc/kattis/problemtools/languages.yaml +RUN mkdir -p /usr/local/artifacts +WORKDIR /usr/local/artifacts +COPY artifacts/deb . +RUN dpkg -i kattis-problemtools*.deb WORKDIR / diff --git a/admin/docker/Dockerfile.icpclangs b/admin/docker/Dockerfile.icpclangs new file mode 100644 index 00000000..c2ce8782 --- /dev/null +++ b/admin/docker/Dockerfile.icpclangs @@ -0,0 +1,21 @@ +# Docker image with all packages needed to run a problemtools .deb, plus +# language support for the "ICPC languages" (C, C++, Java, Kotlin, and Python 3) +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + gcc \ + g++ \ + kotlin \ + openjdk-21-jdk \ + openjdk-21-jre \ + pypy3 diff --git a/admin/docker/Dockerfile.minimal b/admin/docker/Dockerfile.minimal index 534e661f..11b7b414 100644 --- a/admin/docker/Dockerfile.minimal +++ b/admin/docker/Dockerfile.minimal @@ -10,26 +10,11 @@ # PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM ubuntu:22.04 - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt install -y \ - ghostscript \ - libgmpxx4ldbl \ - python-pkg-resources \ - python3-minimal \ - python3-yaml \ - python3-plastex \ - texlive-fonts-recommended \ - texlive-lang-cyrillic \ - texlive-latex-extra \ - texlive-plain-generic \ - tidy - RUN mkdir -p /usr/local/artifacts WORKDIR /usr/local/artifacts COPY artifacts/deb . diff --git a/admin/docker/Dockerfile.runreqs b/admin/docker/Dockerfile.runreqs new file mode 100644 index 00000000..30bc8245 --- /dev/null +++ b/admin/docker/Dockerfile.runreqs @@ -0,0 +1,30 @@ +# Docker image with all packages needed to run a problemtools .deb +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM ubuntu:24.04 + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# Packages required to build and run problemtools +# For libgmp, we technically just need libgmpxx4ldbl here, but for readability +# (and we need libgmp-dev in other images), we take libgmp-dev here +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + dvisvgm \ + fonts-cmu \ + fonts-noto-color-emoji \ + ghostscript \ + libgmp-dev \ + pandoc \ + python3 \ + python3-venv \ + texlive-fonts-recommended \ + texlive-latex-extra \ + texlive-luatex \ + texlive-plain-generic \ + tidy diff --git a/admin/docker/README.md b/admin/docker/README.md new file mode 100644 index 00000000..5e2c6c40 --- /dev/null +++ b/admin/docker/README.md @@ -0,0 +1,25 @@ +Our docker images. Note that images depend on each other, please use the script +admin/update_docker.sh to build images in the correct order. + +We have 4 images which are only used locally in the build process, and are not +uploaded to a repository. + - `runreqs`: Base image containing just the things needed to run problemtools + - `build`: Base image containing just the things needed to build a deb and run problemtools + - `icpclangs`: Base image containing what is needed to run problemtools, plus the "ICPC languages" + - `fulllangs`: Base image containing what is needed to run problemtools, plus all supported languages + +We have 3 images which are meant for end users: + - `minimal`: Image with problemtools installed, but no languages. + - `icpc`: Image with problemtools plus the "ICPC languages" installed. + - `full`: Image with problemtools and all languages + +We have 1 image which is used in our CI (to speed up things - it takes a few +minutes to apt-get install all languages and runtime requirements): + - `githubci`: Image with all languages and everything needed to build a deb and run problemtools + +Build dependencies: +``` + runreqs -> icpclangs -> fullangs -> githubci + / \ | | + build minimal icpc full +``` diff --git a/admin/make_release.sh b/admin/make_release.sh index 0343ac27..a8e47732 100755 --- a/admin/make_release.sh +++ b/admin/make_release.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash # -# Uses git flow and gbp tools (available through Ubuntu packages -# git-flow, git-buildpackage) +# Uses gbp (available through Ubuntu package git-buildpackage) set -e +ALLOW_DIRTY=false +while getopts "d" opt; do + case $opt in + d) ALLOW_DIRTY=true ;; + \?) echo "Invalid option: -$opt" ;; + esac +done + ROOT=$(readlink -f $(dirname $0)/..) VERSION=1.$(date +%Y%m%d) @@ -16,20 +23,54 @@ if [ "$(git tag -l v$VERSION)" != "" ]; then VERSION=$VERSION-rev$REV fi -set -x -git flow release start --showcommands $VERSION +# Steps: +# Pick a version (done by the above loop) +# Update debian/changelog using gbp +# Create and merge pull request with the updated debian/changelog +# Create a github release (using web UI) or gh +# Push to pypi using github action +# Push to docker using admin/update_docker.sh v$VERSION + +CHANGELOG_VERSION=$(dpkg-parsechangelog -l $ROOT/debian/changelog --show-field Version) +if [[ $CHANGELOG_VERSION == $VERSION ]]; then + echo "Debian changelog seems updated" +else + echo "Updating debian changelog (this is surprisingly slow)" + EMAIL=$(git config user.email) gbp dch $ROOT --release --new-version=$VERSION --ignore-branch --git-author --debian-tag='v%(version)s' --debian-branch=release/$VERSION --spawn-editor=never + echo "Please commit the updated changelog, do a pull request, and get it merged, then run this script again on an up-to-date master branch" + echo " git checkout -b release_$VERSION" + echo " git add debian/changelog" + echo " git commit -m 'Update debian changelog for release $VERSION'" + echo " git git push --set-upstream origin release_$VERSION" + exit 0 +fi -# Update _version.py -$ROOT/admin/update_version.py.sh $VERSION +cd $(dirname $(readlink -f $0)) -# Update debian/changelog -gbp dch $ROOT --release --new-version=$VERSION --git-author --debian-tag='v%(version)s' --debian-branch=release/$VERSION --spawn-editor=never +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi -git add $ROOT/problemtools/_version.py $ROOT/debian/changelog -git commit -m "Release of version $VERSION: bump version in problemtools/_version.py and debian/changelog" +GITTAG=master +if [[ $(git rev-parse --abbrev-ref HEAD) != ${GITTAG} && $(git describe --exact-match --tags 2>/dev/null) != ${GITTAG} ]]; then + echo "Repository is currently not on branch/tag ${GITTAG}." + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +THIS_REPO_VERSION=$(git -C $(dirname -- "$0") rev-parse HEAD) +UPSTREAM_VERSION=$(git -C $(dirname -- "$0") ls-remote upstream master | cut -f1) +if [[ $THIS_REPO_VERSION != $UPSTREAM_VERSION ]]; then + echo "Warning: git head of repo does not match upstream. You likely want to update this repo" + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi -git flow release finish --showcommands --message "Release $VERSION" $VERSION -echo "After pushing changes to GitHub, please run" -echo " $ROOT/admin/update_docker.sh v$VERSION" +echo "Create a draft release on github" +echo gh -R Kattis/problemtools release create -d v$VERSION +echo "After finalizing the release on GitHub, please:" +echo " - trigger the pypi release workflow (Run workflow on https://github.com/Kattis/problemtools/actions/workflows/pypi.yml )" +echo " - git fetch upstream (pulling the tag to the local repo)" +echo " - run $ROOT/admin/update_docker.sh v$VERSION" diff --git a/admin/update_docker.sh b/admin/update_docker.sh index ce7e64e2..fe5d2aba 100755 --- a/admin/update_docker.sh +++ b/admin/update_docker.sh @@ -1,51 +1,98 @@ #!/bin/bash set -e -TAG=develop +ALLOW_DIRTY=false +GITTAG=master +DOCKERTAG=develop UPDATE_LATEST=false + +while getopts "d" opt; do + case $opt in + d) ALLOW_DIRTY=true ;; + \?) echo "Invalid option: -$opt" ;; + esac +done + +shift $((OPTIND-1)) + if [ "$1" != "" ]; then - TAG=$1 + GITTAG=$1 + DOCKERTAG=$1 UPDATE_LATEST=true fi - cd $(dirname $(readlink -f $0))/docker -set -x - -# Make the build image and extract build artifacts -# =============================================== -sudo docker build \ - -f Dockerfile.build \ - -t problemtools/build:${TAG} \ - --no-cache \ - --build-arg PROBLEMTOOLS_VERSION="${TAG}" \ - . + +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +if [[ $(git rev-parse --abbrev-ref HEAD) != ${GITTAG} && $(git describe --exact-match --tags 2>/dev/null) != ${GITTAG} ]]; then + echo "Repository is currently not on branch/tag ${GITTAG}." + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +echo "Updating Ubuntu base image" +docker pull ubuntu:24.04 + +# Make our internal images, and our githubci image. Order is important, images depend on each other +echo "Building intermediate images, plus githubci image" +for IMAGE in runreqs build icpclangs fulllangs githubci; do + docker build \ + -f Dockerfile.${IMAGE} \ + -t problemtools/${IMAGE}:${DOCKERTAG} \ + --build-arg PROBLEMTOOLS_VERSION="${DOCKERTAG}" \ + . +done + + +echo "Building deb" mkdir -p artifacts -rm -rf artifacts/deb/* -sudo docker run --rm -v "$(pwd)/artifacts/:/artifacts" problemtools/build:${TAG} cp -r /usr/local/problemtools_build/deb /artifacts +sudo rm -rf artifacts/deb +# Use our build image to build a deb +docker run --rm -v "$(pwd)/../..:/problemtools" -v "$(pwd)/artifacts/deb:/artifacts" problemtools/build:${DOCKERTAG} \ + /bin/bash -c " + set -e ; + mkdir /build ; + cd /build ; + git config --global --add safe.directory /problemtools/.git ; + git clone --branch ${GITTAG} /problemtools ; + cd problemtools ; + make builddeb ; + cp ../*.deb /artifacts" sudo chown -R $USER:$USER artifacts/ -# Build the actual problemtools images -# =============================================== -for IMAGE in minimal icpc full; do - sudo docker build\ - -f Dockerfile.${IMAGE}\ - -t problemtools/${IMAGE}:${TAG}\ - --build-arg PROBLEMTOOLS_VERSION=${TAG}\ - . - if [ "$UPDATE_LATEST" = "true" ]; then - sudo docker tag problemtools/${IMAGE}:${TAG} problemtools/${IMAGE}:latest - fi -done +echo "Testing deb" +if ! docker run --rm -t -v "$(pwd)/../..:/problemtools" -v "$(pwd)/artifacts/deb:/artifacts" problemtools/fulllangs:${DOCKERTAG} \ + /bin/bash -c ' + set -e ; + shopt -s extglob ; + dpkg -i /artifacts/kattis-problemtools* ; + verifyproblem /problemtools/examples/!(README.md)'; then + echo Running verifyproblem on all examples failed. Please review output above to debug.; + exit 1 +fi +echo Tests pass -# Push to Docker Hub -# =============================================== -sudo docker login +echo "Building complete images with problemtools baked in" for IMAGE in minimal icpc full; do - sudo docker push problemtools/${IMAGE}:${TAG} - if [ "$UPDATE_LATEST" = "true" ]; then - sudo docker push problemtools/${IMAGE}:latest - fi + docker build \ + -f Dockerfile.${IMAGE} \ + -t problemtools/${IMAGE}:${DOCKERTAG} \ + --build-arg PROBLEMTOOLS_VERSION="${DOCKERTAG}" \ + . done + + +if [ "${UPDATE_LATEST}" = "true" ]; then + echo "Build complete. If you are happy with the images, run the following:" + for IMAGE in minimal icpc full githubci; do + echo " docker tag problemtools/${IMAGE}:${DOCKERTAG} problemtools/${IMAGE}:latest" + echo " docker push problemtools/${IMAGE}:${DOCKERTAG}" + echo " docker push problemtools/${IMAGE}:latest" + done +fi diff --git a/bin/.run_in_venv.sh b/bin/.run_in_venv.sh new file mode 100755 index 00000000..dce93afb --- /dev/null +++ b/bin/.run_in_venv.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Helper script for the other wrapper scripts to check that a venv exists, or +# give a helpful message if it doesn't. + +VENVPATH="$(dirname "$(dirname "$(readlink -f "$0")")")/venv" + +if [ ! -x "$VENVPATH/bin/python" ]; then + echo "I could not find a python venv at $VENVPATH." + echo "To use these wrapper scripts, please set up a venv by:" + echo " cd $(dirname "$VENVPATH")" + echo " python3 -m venv venv" + echo " venv/bin/pip install -r requirements.txt" + exit 1 +fi + +export PYTHONPATH +PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" +exec "$VENVPATH/bin/python" -m "$@" diff --git a/bin/problem2html.sh b/bin/problem2html.sh index eaa9d2d9..4068fbb3 100755 --- a/bin/problem2html.sh +++ b/bin/problem2html.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.problem2html "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.problem2html "$@" diff --git a/bin/problem2pdf.sh b/bin/problem2pdf.sh index 949c11e8..8995152d 100755 --- a/bin/problem2pdf.sh +++ b/bin/problem2pdf.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.problem2pdf "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.problem2pdf "$@" diff --git a/bin/verifyproblem.sh b/bin/verifyproblem.sh index 364d70c8..48ce5407 100755 --- a/bin/verifyproblem.sh +++ b/bin/verifyproblem.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.verifyproblem "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.verifyproblem "$@" diff --git a/debian/changelog b/debian/changelog index 9891d94a..7f7e9719 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,435 @@ +kattis-problemtools (1.20251128) noble; urgency=medium + + [ Matistjati ] + * Fix multithreading crash + * Better message if judge answer file gets judge error + * Robuster finding judge error tc path + * Cleaner testcase path extraction + + [ matistjati ] + * Fuzz input validators for spaces at end of lines + * Slow fuzzing of output vals + + [ Matistjati ] + * Only fuzz on samples + * Fix type hint + * More crash cases and limit if using python + * Fix type error + * Multipass validation works + * Fix mypy error + * Fix scoring rules and multipass samples + * Remove empty samples warning for multipass + * Multiple output validator fix + * Handle multipass samples for md -> html + * More deterministic input fuzzing + + [ matistjati ] + * Sanity check the parsed author names + + [ Matistjati ] + * Make oddecho worth 100 points + * Reuse testcases using symlinks in oddecho + + [ Gunnar Kreitz ] + * Rewrite latex template handling to not require write access to parent directory + * Let Template.get_file_name() return a Path + * Allow .timelimit, .git, and .gitignore files in problem packages #354 + * Add --version flag to command line tools + * Repair plainproblemname functionality in legacy format (and add a test) + * Add a warning for what looks like latex in problem name + * Replace pdflatex with lualatex. Support emojis. + * Temporarily install new dependencies (to be removed when PR is merged and githubci image is updated) + * Add test coverage of default_validator + * Deal with binary user output and messages in default_validator tests + * Truncate long tokens in default_validator's judge message. #346 Add more test coverage + * Load language config from parent of problem dir, if present #178 + * Add translations for strings related to sample I/O in Swedish and Icelandic #150 + * Fix interactive problem time limit handling #302 + * Support time limits being float (in supporting functions, we still call with integers) + * Implement 2023-07 time limit computation (but hard coded to use accepted for lower bound) + * Refactor time limit computation, making it a bit easier to find and follow + * Use updated, more permissive, regex for file and directory names + * Let make_release print git commands for pushing the changelog for a release. + + -- Gunnar Kreitz Fri, 28 Nov 2025 13:58:24 +0100 + +kattis-problemtools (1.20250606) noble; urgency=medium + + * Fix sample missing from rendered problem statements + + -- Gunnar Kreitz Fri, 06 Jun 2025 21:27:25 +0200 + +kattis-problemtools (1.20250605) noble; urgency=medium + + [ Tagl ] + * Set working directory for submission + + [ Per Austrin ] + * small fixes to README.md + + [ Tagl ] + * Support overwriting directories within submissions with included directories + + [ JoelNiemela ] + * Simplify problem2html argparser and add type annotations + * Simplify problem2pdf argparser and add type annotations + * Add line break for readability + * Add type annotations to verifyproblem + * Use long name for langparam + * Fix potential error + * Use tuple destructuring syntax + * Don't remove argparser_basic_arguments + * Fix CodeFactor errors + + [ Konráð Elí Sigurgeirsson ] + * Update README.md + + [ Joel Niemelä ] + * Update README.md + + [ JoelNiemela ] + * Update kotlin to version 1.8.10 + + [ Harry Zhang ] + * Fix not showing WA test case verifyproblem.py + + [ Gunnar Kreitz ] + * Fix crash in verifyproblem when in or ans files are not utf-8 + + [ Pehr Söderman ] + * Restructure problem2html + * Restructure restrucure problem2pdf + * Improve debugging output + * Parse args properly. + * Reintroduce the get_subgroup, as it is needed for addproblem. + * Reintroduce get_attachment_paths, as it is needed for addproblem. + + [ Tobias Meggendorfer ] + * Improved logging, fix unicode error, include_dir buildrun + * Guard better agains non-unicode feedback + * Revert delayed import + * Fix missed init, add test case + + [ Maarten Sijm ] + * Explicitly set language versions for Java and Kotlin in languages.yaml + + [ Tobias Meggendorfer ] + * Restore old format + + [ Gunnar Kreitz ] + * Add github action to run pytest and flake8 + * Remove travis config, point badges to github badge + * Check that all symlinks point to something existing within the problem package + * Forbid all absolute symlinks + + [ Simon Lindholm ] + * Type annotation fixes + * Add -j flag for multi-threaded validation + * Compile submissions early, improve cleanup + + [ Pehr Söderman ] + * Adding sanity checks for file sizes. + * Add UUID as an optional field in problem.yaml + * Include examples in the manifest + * Fixing running of tests + * Build debian packages in ci + * Cache and add packages + * Warn if sample is empty + + [ Fredrik Niemelä ] + * Change team to user for default validator + + [ Pehr Söderman ] + * Add a dependency on dvisvgm, which was missing + + [ matistjati ] + * Add markdown support + * Added display math + * Add dependencies for markdown + * Style markdown tables + * Remove temp files + * Statement fix + * Some refactoring + * Added image support in markdown + * Added footnote support + * Code cleanup + * md -> html works + + [ Gunnar Kreitz ] + * Remove non-standard judgeerror.txt from example problems + + [ matistjati ] + * Make md styling more constistent with latex + + [ Pehr Söderman ] + * Bump the language versions for c and c++ + * Bump the language versions for Java + + [ matistjati ] + * md->pdf and Reorganize code + + [ Pehr Söderman ] + * GCC should use gnu17 + + [ matistjati ] + * Better md->pdf tables + * Interactive samples for pdf + * Remove bplusa + * PDF problem name + * Add dependencies + * Add problem names + * Added problem name to test hello package + * Improve security by running pandoc without shell capabilities + * Refactoring + * Even more refactoring + * Remove python3-markdown dependency + * Add problem id to pdf and small fixes + + [ Pehr Söderman ] + * Update languages.yaml + + [ Tagl ] + * Run interactive validation with submission's working directory + + [ Matistjati ] + * Change Rust compilation flags + + [ Hugo Söderbergh ] + * Remove deprecated functionality + * add build to .gitignore + + [ JoelNiemela ] + * Add special case error message when user output file is empty + * Modify error message according to github comment + + [ Hugo Söderbergh ] + * add command-line argument, begin generalizing Problem class + * small fix + * abstract problems further + * catch general exception for detecting problem-format + * Add some documentation + * New abstraction, ProblemPart which makes it easier to implement parts of problems + * Problem is no longer an abstract class + * ProblemStatement now exists for old and new format + * Add TODO for ProblemStatement + * Fix issues with ProblemStatement + * Add some documentation and some final fixes + * Small change + * Allow to give class-type for part in Problem.get + * Whoops small bug crashed code + * Fix bug that crashed multithreading for testcase-validation + * Mark ProblemPart.depends_on() as staticmethod + + [ Matistjati ] + * Disable html + * Change to wikimedia example image + * Sanitize image sources + * Remove SVG dependency + * Better markdown styling + * Better sample styling + * Add \nextsample and \remainingsamples + * Better pdf error handling + * Use {{nextsample}} instead of \nextsample + * Relax image checking (implied by global regex on filenames) + * Add svg dependency + + [ Gunnar Kreitz ] + * Explicitly install build-essential, as deb building blows up on it not being installed + + [ Hugo Söderbergh ] + * fix issues with PR + + [ Gunnar Kreitz ] + * Remove test of verifyproblem.generators (which has been removed) + + [ Hugo Söderbergh ] + * Remove bad break-statement and increase consistency in dictionary access + * more concise regex + * Make Problem constructor default to legacy format + * make tests pass + + [ Matistjati ] + * Add back warning/error logging + + [ Gunnar Kreitz ] + * Add mypy to github workflow + * Change type from list to tuple, helping mypy and being clearer + * Fix name of exception (old one also worked, as parser does import * from Scanner, but felt weird) + * Add type annotations and abstract class markers + * Add getProblemPart for when we need to access problem.classes + * Add python tooling files (and vim swp files) to gitignore + * Fix signatures of run in VIVA and checktestdata to match superclass + * Fix/ignore type errors to let mypy catch errors everywhere but generatedata.py + + [ ElliotRipa ] + * Make cls templates able work with either problem format + * Allow problem statement to use either problem format + * Make template.py detect format version instead + * Provisional updates + * Add formatversion.py + * Minor fixes in imports + * Move version specific functionality to separate file + * Change to flag '-v' for format-version + * Add missing parentheses + * Use dictionary instead of data objects for format data + * Make problem2html.py use -v to specify format version + * Add constants for version names + * Rollback problemset_0.1.cls + * Move initialisation of FORMAT_DATA to setup + * Make formatversion.py use dataobjects instead of dicts + * Fix documentation + * Remove unnecessary initialisation + + [ Matistjati ] + * Start sanitization + apply feedback + * Better sanitization + lots of tests + * problem_statement -> statement + * Better md -> pdf sample rendering + * Another escape + * More careful with images + * Make samplexss more focused + * Experimentally reuse normal LaTeX rendering + * Use problemtools problem2pdf to handle md -> pdf + * Cleanup + * librsvg out of focus for this PR + * Ensure nh3 + * Remove ghostscript sanitization. If it wasn't used before, it probably isn't needed + * Add nh3 to deb build + * Linting + * Add back ghostscript sanitization + * Remove unnecessary test + + [ Gunnar Kreitz ] + * Add make clean to clean up support and the mess left by setuptools + * Change debian packaging to dh_virtualenv + * Update readme, adjusting installation instructions so we can use pip dependencies + * Convert from setup.py to pyproject.toml (and use setuptools-scm for versioning) + * Hook sdist to make python -m build work on a clean checkout + * Update wrapper scripts and README + * Update CI workflow to match readme for build requirements (plus build-essentials) + * Force dh_virtualenv to use builtin venv (debugging CI crash) + * Restructure CI/CD to separate deb building from python unit tests + * Stop exposing __version__, users should use importlib.metadata.version instead + * Hardcode path to python, as dh_virtualenv fails to discover it in CI + * Clean up version parsing. Accept 2023-07-draft and 2023-07 version strings + * Add pydantic models for parsing problem.yaml + * Limit problem.yaml config to only system defaults + * Use new metadata parsing mechanism, and start parsing config for 2023-07. + * Bump python version to 3.11 + * Move tests to outside of the package + * Update manifest to include tests support files in sdist, and remove some clutter + * Remove old hack for plasTeX argument (we require >= 3.0 now) + * Clean up incompletely removed plastex_escape hack. Remove unused variable + * Clean up unused variables, old io import, and multiple commands on lines + * Clean up unused import and comparison with None + * Ruff format + * Clean up imports + * Fix minor things flagged by ruff + * Remove unused variables in tests + * Add ruff configuration + * Apply ruff formatting + * Add ruff pre-commit hook + * Replace flake8 with ruff (both linting and formatting) + * Fix incorrect formatting of pydantic errors + * Move is_interactive and is_scoring to be read from problem metadata directly + * Let Problem read and store problem format information. Warn about incomplete 2023-07 support + * Fix validator discovery for 2023-07. Run through all validation for 2023-07 (even if broken) + + [ Matistjati ] + * Add nh3 as dependency + * Fix test import path + * Apply ruff formatting + * More robust footnote finding + * Don't double-escape HTML in samples + * Ghostscript fixes and tests + + [ Gunnar Kreitz ] + * Fix loading a problem with empty problem.yaml and with no statements + * Add utility method to load problem metadata, including names from statements when needed + * Use load_metadata in verifyproblem. Add temporary fallback conf to fix crashes when failing to load metadata + * Use load_metadata in statement_util + * Add apt-get update in workflow to unbreak CI + + [ Matistjati ] + * Convert some example problems to 2023-07-draft + * Add uuid to guess and oddecho + * Better formatting and error for output_validators + + [ Pehr Söderman ] + * Add missing build requirements to debian build + * Update pyproject + + [ Matistjati ] + * Remove now-duplicated import + + [ Gunnar Kreitz ] + * Remove (AFAICT, broken) support for ancient tex statements (0.1) + * Fix bug where we complained about missing show_test_data_groups for non-legacy + * Default language to en. Remove unused --format-version + * Pass Template a filename to render, and pass that through to the latex template + * Rename problem.md to problem.en.md in tests to follow 2023-07-draft + * Refactor of rendering: unify statement finding code, and use Path more + * Use statement_util to find statements. Add more checks. Try rendering even when there are multiple statements in a language + * Make mypy more picky, also checking PlasTeX usage + * Simplify temporary file usage in markdown -> pdf flow + * Fix bug where problem2html cd:s to bad directory, crashing validation of multiple problems + + [ Pehr Söderman ] + * Update link to kattis controlled domain. + + [ Gunnar Kreitz ] + * Replace formatversion.FormatData with a StrEnum + * Add some documentation in the readme regarding current state of format versions + * Add colorlog to get colors for warnings and errors #312 + * Add Swedish problem names + * Fix the logging plasTeX destroys + * Remove accidental commit + * Fix misleading error when missing problem statemetns + * Improve image handling in markdown statements + * Change URL to one that passes filename suffix filter + * Restructure error counters. Fix errors happening prior to check being ignored in count. + * Fix --bail_on_error and --werror being ignored before check + * Refactor problem loading so we can do fatal errors in setup + * Check file and directory names per standard + * Make missing/compilation failure in grader/output validator fatal + * Restore old API for accessing parts of a problem. Simplify part setup + * Change type of attachments.attachments from list[str] to list[Path] + * Convert Problem.metadata to a property to align better with other naming + * Expose computed timelim + * Add back problemtools.run.get_tool_path to API + * Large restructure of how our docker images are built. + * Remove old Dockerfile in root, unused afaict + * Add marker to let mypy use our type annotations + * Replace authors with Kattis AB (pypi only shows one). Set required python version + * Add script to build packages for pypi + * Fix incorrect error when verifying different. Add helpful hint when directory for wrong version exists + * Use problemtools/githubci image in workflow. Run verifyproblem on all examples. + * Check for incompatible types. Warn for unimplemented types + * Check format of interaction samples. #277 Don't warn about empty sample when it contains interactions. + * Add type methods for all types. Add convenience methods on Problem for easier access + * Improve warning for non-standard output validator languages #258 + * Remove generatedata (never made it into the standard) + * Fix pytest dropping a guess.pdf in working directory. Check PDF magic bytes + * Add -d flag to update_docker to allow easier testing locally + * Fix broken git clone command (`${TAG}` expanded to empty string) + * Fix silly error in docker file, causing apt-get update not to run + * Add -d option to allow building in a dirty rep (to facilitate development of build scripts) + * Workflow that builds and pushes a package to testpypi + * Fix version computation when we build pypi packages + * Fix syntax error in github workflow file + * Fix bug where we crashed if we attempted to load/check twice + * Error if problem name exits in a language without a statement + * Add utility function uses_default_validator for output validation. Warn/error on multiple validators + * Fix new mypy error in mypy 1.16 + * Fix missing support for imgbasedir in md2html + * Fix typo in Dockerfile.full causing it to lack a lot of languages + * Remove year from license - IANAL, but AFAICT it's not needed + * Add warning to pypi package script pointing to the github action now that that's set up + * Change update_docker to default to building from master (but keep :develop tag on docker) + * Add convenient way to run docker. Document need to install languages. + + -- Gunnar Kreitz Thu, 05 Jun 2025 10:59:57 +0200 + kattis-problemtools (1.20231016) jammy; urgency=medium [ Don-Khue Le ] diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control index 0a635887..564ed283 100644 --- a/debian/control +++ b/debian/control @@ -2,13 +2,13 @@ Source: kattis-problemtools Section: devel Priority: optional Maintainer: Per Austrin -Build-Depends: debhelper (>= 8.0.0), g++ (>= 4.8), dh-python, python3, python3-setuptools, python3-pytest, python3-yaml, python3-setuptools, python3-pytest, libboost-regex-dev, libgmp-dev, automake, autoconf +Build-Depends: debhelper-compat (= 13), g++ (>= 4.8), dh-virtualenv, python3, libboost-regex-dev, libgmp-dev, automake, autoconf, git, python3-venv, dpkg-dev Standards-Version: 3.9.4 Homepage: https://github.com/Kattis/problemtools Package: kattis-problemtools Architecture: any -Depends: ${shlibs:Depends}, ${python3:Depends}, ${misc:Depends}, python3-plastex, python3-pkg-resources, texlive-plain-generic, texlive-fonts-recommended, texlive-latex-extra, texlive-lang-cyrillic, tidy, ghostscript +Depends: ${shlibs:Depends}, ${misc:Depends}, pandoc, python3, texlive-plain-generic, texlive-fonts-recommended, texlive-latex-extra, texlive-luatex, tidy, ghostscript, dvisvgm, fonts-cmu, fonts-noto-color-emoji Recommends: gcc, g++ Description: Kattis Problem Tools These are tools to manage and verify problem packages in the diff --git a/debian/kattis-problemtools.links b/debian/kattis-problemtools.links new file mode 100644 index 00000000..39354ac1 --- /dev/null +++ b/debian/kattis-problemtools.links @@ -0,0 +1,3 @@ +opt/venvs/kattis-problemtools/bin/verifyproblem usr/bin/verifyproblem +opt/venvs/kattis-problemtools/bin/problem2pdf usr/bin/problem2pdf +opt/venvs/kattis-problemtools/bin/problem2html usr/bin/problem2html diff --git a/debian/rules b/debian/rules index 76a72e36..2c10725f 100755 --- a/debian/rules +++ b/debian/rules @@ -4,12 +4,17 @@ # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 -# Uncomment this to turn off cleanup. -export PYBUILD_DISABLE=clean +%: + dh $@ --with python-virtualenv -export PYBUILD_AFTER_CLEAN=make -C support distclean -export PYBUILD_TEST_PYTEST=1 -export no_proxy=github.com +override_dh_virtualenv: + dh_virtualenv --builtin-venv --python /usr/bin/python3 -%: - dh $@ --with python3 --buildsystem=pybuild +override_dh_strip: + dh_strip --exclude=/PIL/ --exclude=/pillow.libs/ + +override_dh_shlibdeps: + dh_shlibdeps -X/x86/ -X/PIL/.libs/ -X/pillow.libs/ + +override_dh_dwz: + dh_dwz --exclude=/PIL/ --exclude=/pillow.libs/ diff --git a/examples/README.md b/examples/README.md index 2f6107a3..67771232 100644 --- a/examples/README.md +++ b/examples/README.md @@ -25,3 +25,5 @@ more than one language. This is an example of a *scoring* problem where submissions can get different scores depending on which test groups they solve. It also demonstrates how an input validator might check different constraints for different test groups. +In addition, it reuses testcases across test groups using symlinks. +The swedish statement showcases how to use images, footnotes and tables in Markdown. diff --git a/examples/different/output_validators/different_validator/validate.h b/examples/different/output_validators/different_validator/validate.h index 00c896a7..4f653ff1 100644 --- a/examples/different/output_validators/different_validator/validate.h +++ b/examples/different/output_validators/different_validator/validate.h @@ -56,7 +56,6 @@ const int EXITCODE_AC = 42; const int EXITCODE_WA = 43; const std::string FILENAME_AUTHOR_MESSAGE = "teammessage.txt"; const std::string FILENAME_JUDGE_MESSAGE = "judgemessage.txt"; -const std::string FILENAME_JUDGE_ERROR = "judgeerror.txt"; const std::string FILENAME_SCORE = "score.txt"; #define USAGE "%s: judge_in judge_ans feedback_dir < author_out\n" @@ -107,7 +106,7 @@ void wrong_answer(const std::string &msg, ...) { void judge_error(const std::string &msg, ...) { va_list pvar; va_start(pvar, msg); - vreport_feedback(FILENAME_JUDGE_ERROR, msg, pvar); + vreport_feedback(FILENAME_JUDGE_MESSAGE, msg, pvar); assert(0); } diff --git a/examples/different/problem.yaml b/examples/different/problem.yaml index 279a8acb..c67e0aa3 100644 --- a/examples/different/problem.yaml +++ b/examples/different/problem.yaml @@ -5,6 +5,8 @@ ## Author of the problem (default: null) # author: +name: A Different Problem + ## Where the problem was first used (default: null) source: Kattis # source_url: diff --git a/examples/guess/output_validators/guess_validator/validate.cc b/examples/guess/output_validator/guess_validator/validate.cc similarity index 100% rename from examples/guess/output_validators/guess_validator/validate.cc rename to examples/guess/output_validator/guess_validator/validate.cc diff --git a/examples/guess/output_validators/guess_validator/validate.h b/examples/guess/output_validator/guess_validator/validate.h similarity index 97% rename from examples/guess/output_validators/guess_validator/validate.h rename to examples/guess/output_validator/guess_validator/validate.h index 00c896a7..4f653ff1 100644 --- a/examples/guess/output_validators/guess_validator/validate.h +++ b/examples/guess/output_validator/guess_validator/validate.h @@ -56,7 +56,6 @@ const int EXITCODE_AC = 42; const int EXITCODE_WA = 43; const std::string FILENAME_AUTHOR_MESSAGE = "teammessage.txt"; const std::string FILENAME_JUDGE_MESSAGE = "judgemessage.txt"; -const std::string FILENAME_JUDGE_ERROR = "judgeerror.txt"; const std::string FILENAME_SCORE = "score.txt"; #define USAGE "%s: judge_in judge_ans feedback_dir < author_out\n" @@ -107,7 +106,7 @@ void wrong_answer(const std::string &msg, ...) { void judge_error(const std::string &msg, ...) { va_list pvar; va_start(pvar, msg); - vreport_feedback(FILENAME_JUDGE_ERROR, msg, pvar); + vreport_feedback(FILENAME_JUDGE_MESSAGE, msg, pvar); assert(0); } diff --git a/examples/guess/problem.yaml b/examples/guess/problem.yaml index fcb51934..8744f5e7 100644 --- a/examples/guess/problem.yaml +++ b/examples/guess/problem.yaml @@ -1,10 +1,16 @@ +problem_format_version: 2023-07-draft +uuid: 5ca6ba5b-36d5-4eff-8aa7-d967cbc4375e source: Kattis license: cc by-sa -validation: custom interactive +type: interactive +name: + en: Guess the Number + sv: Gissa talet # Override standard limits: say that the TLE solutions provided should # be at least 4 times above the time limit in order for us to be # happy. limits: - time_safety_margin: 4 + time_multipliers: + time_limit_to_tle: 4 diff --git a/examples/guess/problem_statement/problem.en.tex b/examples/guess/statement/problem.en.tex similarity index 100% rename from examples/guess/problem_statement/problem.en.tex rename to examples/guess/statement/problem.en.tex diff --git a/examples/guess/statement/problem.sv.md b/examples/guess/statement/problem.sv.md new file mode 100644 index 00000000..9c49030c --- /dev/null +++ b/examples/guess/statement/problem.sv.md @@ -0,0 +1,20 @@ +Jag tänker på ett hemligt tal mellan $1$ and $100$, kan du gissa vilket? +Givet en gissning kommer jag att berätta om din gissning +var för stor, för liten eller rätt. Du får bara $10$ gissningar, använd +dem klokt! + + +# Interaktion +Ditt program ska skriva ut gissningar om talet. +En gissning är en rad som enbart innehåller ett heltal mellan $1$ och $1000$. +Efter varje gissning måste du flusha standard out. + +Efter varje gissning kan du läs svaret på standard in. +Detta svar är ett av tre ord: + +- `lower` om talet jag tänker på är lägre än din gissning, +- `higher` om talet jag tänker på är högre än din gissning, eller +- `correct` om din gissning är korrekt. + +Efter att ha gissat rätt ska du avsluta ditt program. +Om du gissar fel $10$ gånger får du inga fler chanser och ditt program kommer avbrytas. diff --git a/examples/hello/problem.yaml b/examples/hello/problem.yaml index 194b060f..bc12a981 100644 --- a/examples/hello/problem.yaml +++ b/examples/hello/problem.yaml @@ -1,5 +1,6 @@ source: Kattis license: public domain +name: Hello World! # Fix memory limit at 512 MB. (Note that for most problems, this # should not be done. It is only done in this case because we include diff --git a/examples/hello/submissions/accepted/hello.rs b/examples/hello/submissions/accepted/hello.rs new file mode 100644 index 00000000..47ad8c63 --- /dev/null +++ b/examples/hello/submissions/accepted/hello.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello World!"); +} diff --git a/examples/oddecho/data/secret/subtask1/testdata.yaml b/examples/oddecho/data/secret/subtask1/testdata.yaml index 6a4de612..d6c6f726 100644 --- a/examples/oddecho/data/secret/subtask1/testdata.yaml +++ b/examples/oddecho/data/secret/subtask1/testdata.yaml @@ -1,3 +1,5 @@ -grading: - score: 1 - aggregation: min +on_reject: break +accept_score: 50 +range: 0 50 +grader_flags: min +input_validator_flags: nFive=1 diff --git a/examples/oddecho/data/secret/subtask2/1.ans b/examples/oddecho/data/secret/subtask2/1.ans deleted file mode 100644 index bf8d586a..00000000 --- a/examples/oddecho/data/secret/subtask2/1.ans +++ /dev/null @@ -1,3 +0,0 @@ -a -c -e diff --git a/examples/oddecho/data/secret/subtask2/1.ans b/examples/oddecho/data/secret/subtask2/1.ans new file mode 120000 index 00000000..331f87f4 --- /dev/null +++ b/examples/oddecho/data/secret/subtask2/1.ans @@ -0,0 +1 @@ +../subtask1/1.ans \ No newline at end of file diff --git a/examples/oddecho/data/secret/subtask2/1.in b/examples/oddecho/data/secret/subtask2/1.in deleted file mode 100644 index bc799ff8..00000000 --- a/examples/oddecho/data/secret/subtask2/1.in +++ /dev/null @@ -1,6 +0,0 @@ -5 -a -b -c -d -e diff --git a/examples/oddecho/data/secret/subtask2/1.in b/examples/oddecho/data/secret/subtask2/1.in new file mode 120000 index 00000000..51b7b8c5 --- /dev/null +++ b/examples/oddecho/data/secret/subtask2/1.in @@ -0,0 +1 @@ +../subtask1/1.in \ No newline at end of file diff --git a/examples/oddecho/data/secret/subtask2/2.ans b/examples/oddecho/data/secret/subtask2/2.ans deleted file mode 100644 index d65cd02c..00000000 --- a/examples/oddecho/data/secret/subtask2/2.ans +++ /dev/null @@ -1,3 +0,0 @@ -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/examples/oddecho/data/secret/subtask2/2.ans b/examples/oddecho/data/secret/subtask2/2.ans new file mode 120000 index 00000000..13e07149 --- /dev/null +++ b/examples/oddecho/data/secret/subtask2/2.ans @@ -0,0 +1 @@ +../subtask1/2.ans \ No newline at end of file diff --git a/examples/oddecho/data/secret/subtask2/2.in b/examples/oddecho/data/secret/subtask2/2.in deleted file mode 100644 index 294867e4..00000000 --- a/examples/oddecho/data/secret/subtask2/2.in +++ /dev/null @@ -1,6 +0,0 @@ -5 -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb -ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc -bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/examples/oddecho/data/secret/subtask2/2.in b/examples/oddecho/data/secret/subtask2/2.in new file mode 120000 index 00000000..6b0f7cbb --- /dev/null +++ b/examples/oddecho/data/secret/subtask2/2.in @@ -0,0 +1 @@ +../subtask1/2.in \ No newline at end of file diff --git a/examples/oddecho/data/secret/subtask2/3.ans b/examples/oddecho/data/secret/subtask2/3.ans deleted file mode 100644 index e05c6300..00000000 --- a/examples/oddecho/data/secret/subtask2/3.ans +++ /dev/null @@ -1,3 +0,0 @@ -testdata -fun -write diff --git a/examples/oddecho/data/secret/subtask2/3.ans b/examples/oddecho/data/secret/subtask2/3.ans new file mode 120000 index 00000000..5efebeff --- /dev/null +++ b/examples/oddecho/data/secret/subtask2/3.ans @@ -0,0 +1 @@ +../subtask1/3.ans \ No newline at end of file diff --git a/examples/oddecho/data/secret/subtask2/3.in b/examples/oddecho/data/secret/subtask2/3.in deleted file mode 100644 index bb472c64..00000000 --- a/examples/oddecho/data/secret/subtask2/3.in +++ /dev/null @@ -1,6 +0,0 @@ -5 -testdata -is -fun -to -write diff --git a/examples/oddecho/data/secret/subtask2/3.in b/examples/oddecho/data/secret/subtask2/3.in new file mode 120000 index 00000000..6927c199 --- /dev/null +++ b/examples/oddecho/data/secret/subtask2/3.in @@ -0,0 +1 @@ +../subtask1/3.in \ No newline at end of file diff --git a/examples/oddecho/data/secret/subtask2/testdata.yaml b/examples/oddecho/data/secret/subtask2/testdata.yaml index 6a4de612..23fad897 100644 --- a/examples/oddecho/data/secret/subtask2/testdata.yaml +++ b/examples/oddecho/data/secret/subtask2/testdata.yaml @@ -1,3 +1,5 @@ -grading: - score: 1 - aggregation: min +on_reject: break +accept_score: 50 +range: 0 50 +grader_flags: min +input_validator_flags: nFive=0 diff --git a/examples/oddecho/input_format_validators/validator/validator.cpp b/examples/oddecho/input_validators/validator/validator.cpp similarity index 100% rename from examples/oddecho/input_format_validators/validator/validator.cpp rename to examples/oddecho/input_validators/validator/validator.cpp diff --git a/examples/oddecho/input_format_validators/validator/validator.h b/examples/oddecho/input_validators/validator/validator.h similarity index 100% rename from examples/oddecho/input_format_validators/validator/validator.h rename to examples/oddecho/input_validators/validator/validator.h diff --git a/examples/oddecho/problem.yaml b/examples/oddecho/problem.yaml index f2556316..06b2e7df 100644 --- a/examples/oddecho/problem.yaml +++ b/examples/oddecho/problem.yaml @@ -1,5 +1,9 @@ +problem_format_version: 2023-07-draft +uuid: 025dfeea-eb85-4532-94d1-3108ec03c80f license: cc by-sa -author: Johan Sannemo +credits: Johan Sannemo source: Principles of Algorithmic Problem Solving type: scoring -name: Echo +name: + en: Odd Echo + sv: Udda eko diff --git a/examples/oddecho/statement/cave.jpg b/examples/oddecho/statement/cave.jpg new file mode 100644 index 00000000..670bbeda Binary files /dev/null and b/examples/oddecho/statement/cave.jpg differ diff --git a/examples/oddecho/problem_statement/problem.en.tex b/examples/oddecho/statement/problem.en.tex similarity index 100% rename from examples/oddecho/problem_statement/problem.en.tex rename to examples/oddecho/statement/problem.en.tex diff --git a/examples/oddecho/statement/problem.sv.md b/examples/oddecho/statement/problem.sv.md new file mode 100644 index 00000000..55f9806f --- /dev/null +++ b/examples/oddecho/statement/problem.sv.md @@ -0,0 +1,33 @@ +**EKO! Eko! Ek...** + +![CC-BY-SA 2.0 By William Craig on wikimedia.org](cave.jpg) + +Du älskar att skrika i grottor för att höra dina ord ekade tillbaka till dig. Tyvärr, som en hårt arbetande mjukvaruingenjör, har du +inte tid för att komma ut och skrika i grottor så ofta. Istället skulle du vilja implementera ett program som fungerar som en ersättning för en grotta. + +Ibland vill du mata in några ord i programmet och få dem ekade tillbaka till dig. Men, som det är välkänt, om du skriker för snabbt i en grotta kan ekot störa de nya ord du säger. [^1] Mer specifikt, vartannat ord du säger kommer att störa ekot av ditt tidigare ord. Därför kommer endast det första, tredje, femte och så vidare ordet faktiskt att producera ett eko. + +Din uppgift är att skriva ett program som simulerar detta beteende. + +## Indata + +Den första raden av indata innehåller ett heltal $N$ ($1 \le N \le 10$). + +De följande $N$ raderna innehåller vardera ett ord. Varje ord är högst $100$ bokstäver långt och innehåller endast bokstäverna `a-z`. + +## Utdata + +Skriv ut de ord som har udda index (dvs. första, tredje, femte och så vidare) i inmatningen. + + +## Poängsättning + +Din lösning kommer att testas på en mängd testfallsgrupper. +För att få poäng för en grupp så måste du klara alla testfall i gruppen. + +| Grupp | Poäng | Begränsningar | +|-------|-------|--------------------------| +| 1 | 1 | $N$ är alltid $5$ | +| 2 | 1 | Inga ytterligare begränsningar | + +[^1]: [https://sv.wikipedia.org/wiki/Interferens](https://sv.wikipedia.org/wiki/Interferens) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..1a4c5a69 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +ignore_missing_imports = False +follow_untyped_imports = True +install_types = True +check_untyped_defs = True +ignore_errors = False diff --git a/problemtools/ProblemPlasTeX/ProblemsetMacros.py b/problemtools/ProblemPlasTeX/ProblemsetMacros.py index 31794c98..b5bd4f41 100644 --- a/problemtools/ProblemPlasTeX/ProblemsetMacros.py +++ b/problemtools/ProblemPlasTeX/ProblemsetMacros.py @@ -1,16 +1,15 @@ import sys import os import os.path -import io from plasTeX.DOM import Node from plasTeX.Base import Command from plasTeX.Base import DimenCommand from plasTeX.Logging import getLogger -import plasTeX.Packages.graphics as graphics log = getLogger() status = getLogger('status') + # Ugly hack: assume textwidth is 600pt. True for Kattis but not in # general. class textwidth(DimenCommand): @@ -25,7 +24,7 @@ def clean_width(width): nodes = width.childNodes if len(nodes) != 2 or nodes[1].nodeName != 'textwidth': return width - return '%.2f%%' % (100*float(nodes[0])) + return '%.2f%%' % (100 * float(nodes[0])) # \problemheader @@ -33,9 +32,8 @@ class problemheader(Command): args = 'title id:str' def invoke(self, tex): - res = Command.invoke(self, tex) - timelimfile = os.path.join(os.path.dirname(tex.filename), - '..', '.timelimit') + super().invoke(tex) + timelimfile = os.path.join(os.path.dirname(tex.filename), '..', '.timelimit') if os.path.isfile(timelimfile): self.attributes['timelim'] = open(timelimfile, 'r').read() @@ -45,10 +43,10 @@ class sampletable(Command): args = 'header1 file1:str header2 file2:str' def read_sample_file(self, filename): - return io.open(filename, 'r', encoding='utf-8').read() + return open(filename, 'r', encoding='utf-8').read() def invoke(self, tex): - res = Command.invoke(self, tex) + super().invoke(tex) dir = os.path.dirname(tex.filename) file1 = os.path.join(dir, self.attributes['file1']) file2 = os.path.join(dir, self.attributes['file2']) @@ -67,28 +65,32 @@ class sampletableinteractive(Command): args = 'header read write file:str' def read_sample_interaction(self, filename): - data = io.open(filename, 'r', encoding='utf-8').read() + data = open(filename, 'r', encoding='utf-8').read() messages = [] - cur_msg = [] + cur_msg: list[str] = [] cur_mode = None for line in data.split('\n'): - if not line: continue - if line[0] == '<': mode = 'read' - elif line[0] == '>': mode = 'write' - else: continue + if not line: + continue + if line[0] == '<': + mode = 'read' + elif line[0] == '>': + mode = 'write' + else: + continue line = line[1:] if mode != cur_mode: - if cur_mode: messages.append({'mode': cur_mode, - 'data': '\n'.join(cur_msg)}) + if cur_mode: + messages.append({'mode': cur_mode, 'data': '\n'.join(cur_msg)}) cur_msg = [] cur_msg.append(line) cur_mode = mode - if cur_mode: messages.append({'mode': cur_mode, - 'data': '\n'.join(cur_msg)}) + if cur_mode: + messages.append({'mode': cur_mode, 'data': '\n'.join(cur_msg)}) return messages def invoke(self, tex): - res = Command.invoke(self, tex) + super().invoke(tex) dir = os.path.dirname(tex.filename) file = os.path.join(dir, self.attributes['file']) try: @@ -104,21 +106,19 @@ def invoke(self, tex): # \includegraphics implementation) class _graphics_command(Command): def invoke(self, tex): - res = Command.invoke(self, tex) + res = super().invoke(tex) # Overcome plasTeX bug by looking for love in the right place + assert self.ownerDocument is not None # Keep mypy happy basetex = self.ownerDocument.userdata['base_tex_instance'] f = self.attributes['file'] - ext = self.ownerDocument.userdata.getPath( - 'packages/graphicx/extensions', - ['.png', '.jpg', '.jpeg', '.gif', '.pdf']) - paths = self.ownerDocument.userdata.getPath( - 'packages/graphicx/paths', [os.path.dirname(basetex.filename)]) - img = None + ext = self.ownerDocument.userdata.getPath('packages/graphicx/extensions', ['.png', '.jpg', '.jpeg', '.gif', '.pdf']) + paths = self.ownerDocument.userdata.getPath('packages/graphicx/paths', [os.path.dirname(basetex.filename)]) + img: str | None = None # Check for file using graphicspath for p in paths: - for e in ['']+ext: - fname = os.path.join(p, f+e) + for e in [''] + ext: + fname = os.path.join(p, f + e) if os.path.isfile(fname): img = os.path.abspath(fname) break @@ -127,14 +127,14 @@ def invoke(self, tex): # Check for file using kpsewhich if img is None: - for e in ['']+ext: + for e in [''] + ext: try: - img = os.path.abspath(basetex.kpsewhich(f+e)) + img = os.path.abspath(basetex.kpsewhich(f + e)) break except (OSError, IOError): pass - if not os.path.isfile(img): + if img is None or not os.path.isfile(img): log.warning('Could not identify image "%s"' % f) self.imageoverride = img @@ -147,17 +147,20 @@ class illustration(_graphics_command): def invoke(self, tex): res = _graphics_command.invoke(self, tex) - self.style['width'] = '%.2f%%' % (100*self.attributes['width']) + self.style['width'] = '%.2f%%' % (100 * self.attributes['width']) return res + # Dummy for \fontencoding to suppress warnings class fontencoding(Command): args = 'charset:str' + # Dummy for \selectfont to suppress warnings. class selectfont(Command): pass + # Dummy for \ExecuteOptions to suppress warnings. class ExecuteOptions(Command): pass diff --git a/problemtools/ProblemPlasTeX/__init__.py b/problemtools/ProblemPlasTeX/__init__.py index f0a608e7..b136ab7a 100644 --- a/problemtools/ProblemPlasTeX/__init__.py +++ b/problemtools/ProblemPlasTeX/__init__.py @@ -2,7 +2,6 @@ import os import shutil import subprocess -import plasTeX.Renderers from plasTeX.Renderers.PageTemplate import Renderer from plasTeX.Filenames import Filenames from plasTeX.Imagers import Image @@ -10,15 +9,15 @@ log = getLogger() + # Adapted from plasTeX.Imagers.Imager class class ImageConverter(object): fileExtension = '.png' imageAttrs = '' imageUnits = '' - imageTypes = ['.png', '.jpg', '.jpeg', '.gif'] #, '.svg'] - imageConversion = {'.pdf': ['.png', - ['gs', '-dUseCropBox', '-sDEVICE=pngalpha', '-r300', '-o']]} + imageTypes = ['.png', '.jpg', '.jpeg', '.gif'] # , '.svg'] + imageConversion = {'.pdf': ('.png', ['gs', '-dUseCropBox', '-sDEVICE=pngalpha', '-r300', '-o'])} def __init__(self, document): self.config = document.config @@ -28,10 +27,13 @@ def __init__(self, document): self.staticimages = {} # Filename generator - self.newFilename = Filenames(self.config['images'].get('filenames'), - None, - variables={'jobname': document.userdata.get('jobname', '')}, - extension=self.fileExtension, invalid={}) + self.newFilename = Filenames( + self.config['images'].get('filenames'), + None, + variables={'jobname': document.userdata.get('jobname', '')}, + extension=self.fileExtension, + invalid={}, + ) def close(self): return @@ -55,14 +57,14 @@ def getImage(self, node): if oldext in self.imageConversion: # Need to convert image newext = self.imageConversion[oldext][0] - path = os.path.splitext(path)[0]+newext + path = os.path.splitext(path)[0] + newext cmd = self.imageConversion[oldext][1] + [path, name] status = subprocess.call(cmd) if status: log.warning('Failed to convert %s image "%s to %s', oldext, name, newext) else: # Just copy it - path = os.path.splitext(path)[0]+oldext + path = os.path.splitext(path)[0] + oldext shutil.copyfile(name, path) img = Image(path, self.ownerDocument.config['images']) self.staticimages[name] = img @@ -74,25 +76,25 @@ def getImage(self, node): return None - - class ProblemRenderer(Renderer): - """ Renderer for ProblemHTML documents """ + """Renderer for ProblemHTML documents""" fileExtension = '.html' imageTypes = ['.png', '.jpg', '.jpeg', '.gif'] vectorImageTypes = ['.svg'] - def render(self, document): - templatepaths = [os.path.join(os.path.dirname(__file__), '../templates/html'), - os.path.join(os.path.dirname(__file__), '../../templates/html'), - '/usr/lib/problemtools/templates/html'] + def render(self, document, postProcess=None): + templatepaths = [ + os.path.join(os.path.dirname(__file__), '../templates/html'), + os.path.join(os.path.dirname(__file__), '../../templates/html'), + '/usr/lib/problemtools/templates/html', + ] templatepath = None for p in templatepaths: if os.path.isdir(p): templatepath = p break - if templatepath == None: + if templatepath is None: raise Exception('Could not find templates needed for conversion to HTML') # Ugly but unfortunately PlasTeX is quite inflexible when it comes to @@ -113,8 +115,7 @@ def processFileContent(self, document, s): s = Renderer.processFileContent(self, document, s) # Force XHTML syntax on empty tags - s = re.compile(r'(<(?:hr|br|img|link|meta)\b.*?)\s*/?\s*(>)', - re.I|re.S).sub(r'\1 /\2', s) + s = re.compile(r'(<(?:hr|br|img|link|meta)\b.*?)\s*/?\s*(>)', re.I | re.S).sub(r'\1 /\2', s) # Remove empty paragraphs s = re.compile(r'

\s*

', re.I).sub(r'', s) diff --git a/problemtools/ProblemPlasTeX/graphicx.py b/problemtools/ProblemPlasTeX/graphicx.py index 6fa954d4..e9669397 100644 --- a/problemtools/ProblemPlasTeX/graphicx.py +++ b/problemtools/ProblemPlasTeX/graphicx.py @@ -1,9 +1,10 @@ import plasTeX.Packages.graphics as graphics -from ProblemsetMacros import _graphics_command, clean_width +from problemtools.ProblemPlasTeX.ProblemsetMacros import _graphics_command, clean_width # Reimplementation of graphicx package because plasTeX is broken and # annoying. + class includegraphics(_graphics_command): args = '* [ options:dict ] file:str' packageName = 'graphicx' @@ -21,8 +22,10 @@ def invoke(self, tex): self.style['width'] = clean_width(width) return res + class DeclareGraphicsExtensions(graphics.DeclareGraphicsExtensions): packageName = 'graphicx' + class graphicspath(graphics.graphicspath): packageName = 'graphicx' diff --git a/problemtools/ProblemPlasTeX/import.py b/problemtools/ProblemPlasTeX/import.py index 5005936a..e2d4e14f 100644 --- a/problemtools/ProblemPlasTeX/import.py +++ b/problemtools/ProblemPlasTeX/import.py @@ -6,6 +6,7 @@ log = getLogger() status = getLogger('status') + # (Partial) implementation of import.sty because plasTeX does not ship # with an implementation. Only implement \import command which is the # only one we'll use. diff --git a/problemtools/ProblemPlasTeX/listingsutf8.py b/problemtools/ProblemPlasTeX/listingsutf8.py index 2a2d1f56..d022ef03 100644 --- a/problemtools/ProblemPlasTeX/listingsutf8.py +++ b/problemtools/ProblemPlasTeX/listingsutf8.py @@ -4,23 +4,21 @@ import os import io -import ProblemsetMacros - log = getLogger() # Implementation of (parts) of listingsutf8 package since PlasTeX does # not have one + class lstinputlisting(Command): args = '* [ options:dict ] file:str' - def read_file(self, filename): - data = io.open(filename, 'r', encoding='utf-8').read() - data = ProblemsetMacros.plastex_escape(data) - return data + def read_file(self, filename) -> str: + return io.open(filename, 'r', encoding='utf-8').read() - def invoke(self, tex): - res = Command.invoke(self, tex) + def invoke(self, tex) -> None: + super().invoke(tex) + assert self.ownerDocument is not None # Keep mypy happy basetex = self.ownerDocument.userdata['base_tex_instance'] f = self.attributes['file'] # Maybe more paths to look in? diff --git a/problemtools/ProblemPlasTeX/ulem.py b/problemtools/ProblemPlasTeX/ulem.py index c2d9f79f..f891f86b 100644 --- a/problemtools/ProblemPlasTeX/ulem.py +++ b/problemtools/ProblemPlasTeX/ulem.py @@ -1,11 +1,29 @@ from plasTeX.Base.LaTeX.FontSelection import TextCommand -class uline(TextCommand): pass -class uuline(TextCommand): pass -class uwave(TextCommand): pass -class sout(TextCommand): pass -class xout(TextCommand): pass -class dashuline(TextCommand): pass -class dotuline(TextCommand): pass +class uline(TextCommand): + pass + +class uuline(TextCommand): + pass + + +class uwave(TextCommand): + pass + + +class sout(TextCommand): + pass + + +class xout(TextCommand): + pass + + +class dashuline(TextCommand): + pass + + +class dotuline(TextCommand): + pass diff --git a/problemtools/__init__.py b/problemtools/__init__.py index 8dee4bf8..e69de29b 100644 --- a/problemtools/__init__.py +++ b/problemtools/__init__.py @@ -1 +0,0 @@ -from ._version import __version__ diff --git a/problemtools/_version.py b/problemtools/_version.py deleted file mode 100644 index 98cf094a..00000000 --- a/problemtools/_version.py +++ /dev/null @@ -1,2 +0,0 @@ -# Auto-generated from git changelog, do not edit! -__version__ = '1.20231016' diff --git a/problemtools/config.py b/problemtools/config.py index bcc74896..a85493be 100644 --- a/problemtools/config.py +++ b/problemtools/config.py @@ -1,13 +1,15 @@ import collections import os import yaml +from pathlib import Path +from typing import Mapping class ConfigError(Exception): pass -def load_config(configuration_file): +def load_config(configuration_file: str, priority_dirs: list[Path] = []) -> dict: """Load a problemtools configuration file. Args: @@ -15,42 +17,42 @@ def load_config(configuration_file): relative to config directory so typically just a file name without paths, e.g. "languages.yaml". """ - res = None + res: dict | None = None - for dirname in __config_file_paths(): - path = os.path.join(dirname, configuration_file) + for dirname in __config_file_paths() + priority_dirs: + path = dirname / configuration_file new_config = None - if os.path.isfile(path): + if path.is_file(): try: with open(path, 'r') as config: new_config = yaml.safe_load(config.read()) - except (yaml.parser.ParserError, yaml.parser.ScannerError) as err: - raise ConfigError('Config file %s: failed to parse: %s' % (path, err)) + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as err: + raise ConfigError(f'Config file {path}: failed to parse: {err}') if res is None: if new_config is None: - raise ConfigError('Base configuration file %s not found in %s' - % (configuration_file, path)) + raise ConfigError(f'Base configuration file {configuration_file} not found in {path}') res = new_config elif new_config is not None: __update_dict(res, new_config) + assert res is not None, 'Failed to load config (should never happen, we should have hit an error in loop above)' return res -def __config_file_paths(): +def __config_file_paths() -> list[Path]: """ Paths in which to look for config files, by increasing order of priority (i.e., any config in the last path should take precedence over the others). """ - return [os.path.join(os.path.dirname(__file__), 'config'), - os.path.join('/etc', 'kattis', 'problemtools'), - os.path.join(os.environ.get('XDG_CONFIG_HOME', - os.path.join(os.path.expanduser('~'), '.config')), - 'problemtools')] + return [ + Path(__file__).parent / 'config', + Path('/etc/kattis/problemtools'), + Path(os.environ.get('XDG_CONFIG_HOME', Path.home() / '.config')) / 'problemtools', + ] -def __update_dict(orig, update): +def __update_dict(orig: dict, update: Mapping) -> None: """Deep update of a dictionary For each entry (k, v) in update such that both orig[k] and v are @@ -58,10 +60,8 @@ def __update_dict(orig, update): For all other entries (k, v), orig[k] is set to v. """ - for (key, value) in update.items(): - if (key in orig and - isinstance(value, collections.abc.Mapping) and - isinstance(orig[key], collections.abc.Mapping)): + for key, value in update.items(): + if key in orig and isinstance(value, collections.abc.Mapping) and isinstance(orig[key], collections.abc.Mapping): __update_dict(orig[key], value) else: orig[key] = value diff --git a/problemtools/config/languages.yaml b/problemtools/config/languages.yaml index 79438a0f..f5f57e21 100644 --- a/problemtools/config/languages.yaml +++ b/problemtools/config/languages.yaml @@ -209,7 +209,7 @@ prolog: # Python2 with shebang comes before default python3. python2_with_shebang: - name: 'Python 2' + name: 'Python 2 (w/PyPy)' priority: 860 files: '*.py *.py2' shebang: '^#!.*python2\b' @@ -217,7 +217,7 @@ python2_with_shebang: run: '/usr/bin/env python2 "{mainfile}"' python3: - name: 'Python 3' + name: 'Python 3 (w/PyPy3)' priority: 850 files: '*.py *.py3' compile: '/usr/bin/env python3 -m py_compile {files}' @@ -225,7 +225,7 @@ python3: # Python2 without shebang comes after python3. python2: - name: 'Python 2' + name: 'Python 2 (w/PyPy)' priority: 840 files: '*.py2' compile: '/usr/bin/env python2 -m py_compile {files}' diff --git a/problemtools/config/problem.yaml b/problemtools/config/problem.yaml index 44954162..7b294c07 100644 --- a/problemtools/config/problem.yaml +++ b/problemtools/config/problem.yaml @@ -1,20 +1,9 @@ -type: pass-fail -author: '' -source: '' -source_url: '' -license: unknown -rights_owner: '' - -validation: default -validator_flags: '' - limits: - time_multiplier: 5 - time_safety_margin: 2 memory: 1024 output: 8 code: 128 compilation_time: 60 + compilation_memory: 1024 validation_time: 60 validation_memory: 1024 validation_output: 8 diff --git a/problemtools/formatversion.py b/problemtools/formatversion.py new file mode 100644 index 00000000..c13cae95 --- /dev/null +++ b/problemtools/formatversion.py @@ -0,0 +1,57 @@ +import yaml +from enum import StrEnum +from pathlib import Path + + +class FormatVersion(StrEnum): + LEGACY = 'legacy' + V_2023_07 = '2023-07-draft' # When 2023-07 is finalized, replace this and update _missing_ + + @property + def statement_directory(self) -> str: + match self: + case FormatVersion.LEGACY: + return 'problem_statement' + case FormatVersion.V_2023_07: + return 'statement' + + @property + def statement_extensions(self) -> list[str]: + match self: + case FormatVersion.LEGACY: + return ['tex'] + case FormatVersion.V_2023_07: + return ['md', 'tex'] + + @property + def output_validator_directory(self) -> str: + match self: + case FormatVersion.LEGACY: + return 'output_validators' + case FormatVersion.V_2023_07: + return 'output_validator' + + @property + def submission_directories(self) -> list[str]: + match self: + case FormatVersion.LEGACY: + return ['accepted', 'partially_accepted', 'wrong_answer', 'time_limit_exceeded', 'run_time_error'] + case FormatVersion.V_2023_07: + # TODO: parse submissions.yaml if applicable, since + # 2023-07 and later formats support adding more submission directories + return ['accepted', 'rejected', 'wrong_answer', 'time_limit_exceeded', 'run_time_error', 'brute_force'] + + # Support 2023-07 and 2023-07-draft strings. + # This method should be replaced with an alias once we require python 3.13 + @classmethod + def _missing_(cls, value): + if value == '2023-07': + return cls.V_2023_07 + return None + + +def get_format_version(problem_root: Path) -> FormatVersion: + """Loads the version from the problem in problem_root""" + with open(problem_root / 'problem.yaml') as f: + config: dict = yaml.safe_load(f) or {} + return FormatVersion(config.get('problem_format_version', FormatVersion.LEGACY)) diff --git a/problemtools/generatedata.py b/problemtools/generatedata.py deleted file mode 100644 index 33d0aebf..00000000 --- a/problemtools/generatedata.py +++ /dev/null @@ -1,273 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -import sys -import os -import glob -import tempfile -import shutil -import yaml -from argparse import ArgumentParser -from multiprocessing import Pool, cpu_count - -from .verifyproblem import Generators, ProblemAspect, Problem, is_RTE, argparser_basic_arguments, initialize_logging - -ALL_EXTENSIONS = ['in', 'ans'] + Generators._VISUALIZER_EXTENSIONS - -def argparser() -> ArgumentParser: - parser = ArgumentParser(description='Generate test data for a problem package in the Kattis problem format.') - parser.add_argument('-g', '--generate', - action='store_true', - help='generate test data') - parser.add_argument('-c', '--clean', - action='store_true', - help='clean up generated files') - parser.add_argument('-C', '--clean_all', - action='store_true', - help='clean up generated and unrecognized files') - parser.add_argument('-n', '--dry_run', - action='store_true', - help='don\'t actually do anything') - parser.add_argument('-j', '--parallelism', - type=int, - default=None, - help='level of parallelism') - - argparser_basic_arguments(parser) - - parser.add_argument('problemdir', nargs='+') - return parser - - -def clean(prob, args): - ProblemAspect.errors = 0 - ProblemAspect.warnings = 0 - base_path = os.path.join(prob.probdir, 'data') - - testcases = { case['path']: case for case in prob.generators._testcases } - - def walk(name, path): - case_count = 0 - cases = set() - empty = True - for fname in sorted(os.listdir(path)): - curpath = os.path.join(path, fname) - nice_path = os.path.relpath(curpath, base_path) - if '.' in fname: - fname, ext = fname.split('.', 1) - else: - ext = '' - curname = '%s/%s' % (name, fname) - - if os.path.isdir(curpath): - next_empty, next_cases = walk(curname, curpath) - case_count += next_cases - if next_empty: - if not args.dry_run: - os.rmdir(curpath) - else: - empty = False - else: - remove = args.clean_all - is_case = False - if (fname, ext) == ('testdata', 'yaml'): - if curname + '.yaml' in prob.generators._testdata_yaml: - remove = True - elif curname in testcases: - is_case = True - case = testcases[curname] - if ext == 'in': - remove = not case['manual'] - elif ext == 'ans': - remove = case['solution'] is not None - elif ext in Generators._VISUALIZER_EXTENSIONS: - remove = case['visualizer'] is not None - - if remove: - prob.generators.msg('Removing %s' % nice_path) - if not args.dry_run: - os.unlink(curpath) - if is_case and curname not in cases: - cases.add(curname) - case_count += 1 - else: - empty = False - - return empty, case_count - - cases_cleaned = 0 - for directory in prob.generators._data_directories: - path = os.path.join(base_path, directory) - if os.path.isdir(path): - cases_cleaned += walk('data/%s' % directory, path)[1] - return cases_cleaned, ProblemAspect.errors, ProblemAspect.warnings - - -class GenerateState: - prob = None - args = None - -def generate_case(case_idx): - ProblemAspect.errors = 0 - ProblemAspect.warnings = 0 - prob = GenerateState.prob - args = GenerateState.args - case = prob.generators._testcases[case_idx] - - steps = [ - ('input', True, None, '.in'), - ('solution', False, '.in', '.ans'), - ('visualizer', False, '.in', None), - ] - - try: - tmp_dir = tempfile.mkdtemp(prefix='gencase', dir=prob.tmpdir) - staging_dir = os.path.join(tmp_dir, 'staging') - os.mkdir(staging_dir) - out_dir = os.path.join(*([prob.probdir] + case['path'].split('/')[:-1])) - name = case['path'].split('/')[-1] - ok = args.dry_run or os.path.isdir(out_dir) - for (gen_type, mandatory, in_ext, out_ext) in steps: - if not ok: - break - prog = case.get(gen_type) - if prog is None: - ok = not mandatory - continue - prog, pargs = prog - prog = prob.generators._generators.get(prog) - if prog is None: - ok = not mandatory - continue - - if gen_type == 'input': - prob.generators.msg('Generating %s' % case['path'].replace('data/', '', 1)) - - if isinstance(prog, str): - assert gen_type == 'input' - assert prog.endswith('.in') - for ext in ALL_EXTENSIONS: - path = prog[:-2] + ext - if os.path.isfile(path): - shutil.copyfile(path, os.path.join(staging_dir, '%s.%s' % (name, ext))) - else: - errfile = os.path.join(tmp_dir, 'error') - params = {'args': pargs, 'errfile': errfile} - if in_ext is not None: - params['infile'] = os.path.join(staging_dir, name + in_ext) - if out_ext is not None: - outfile = os.path.join(tmp_dir, 'output') - params['outfile'] = outfile - - oldwd = os.getcwd() - os.chdir(staging_dir) - status, _ = prog.run(**params) - os.chdir(oldwd) - if is_RTE(status): - ok = not mandatory - stderr = None - if os.path.isfile(errfile): - with open(errfile, 'r') as f: - stderr = f.read() - prob.generators.error('Generator of type %s crashed with status %s' % (gen_type, status), stderr) - continue - - if out_ext is not None: - dest = os.path.join(staging_dir, name + out_ext) - if not os.path.isfile(dest): - shutil.copyfile(outfile, dest) - if ok: - for fname in os.listdir(staging_dir): - if '.' not in fname: - continue - curname, ext = fname.split('.', 1) - if curname == name and ext in ALL_EXTENSIONS: - fpath = os.path.join(staging_dir, fname) - if os.path.isfile(fpath) and not args.dry_run: - shutil.copyfile(fpath, os.path.join(out_dir, fname)) - return ok, ProblemAspect.errors, ProblemAspect.warnings - finally: - shutil.rmtree(tmp_dir) - - -def generate(prob, args): - # Create directory structure - created = set() - for case in prob.generators._testcases: - path = os.path.join(*([prob.probdir] + case['path'].split('/')[:-1])) - if path not in created: - created.add(path) - if not os.path.isdir(path) and not args.dry_run: - try: - os.makedirs(path) - except Exception as e: - prob.generators.error('Could not create path %s' % path, e) - - # Populate testdata.yaml files - for path, content in prob.generators._testdata_yaml.items(): - prob.generators.msg('Generating %s' % path.replace('data/', '', 1)) - path = os.path.join(*([prob.probdir] + path.split('/'))) - if not args.dry_run: - try: - with open(path, 'w') as f: - yaml.dump(content, f) - except Exception as e: - prob.generators.error('Could not write %s' % path, e) - - # Generate test cases in parallel - GenerateState.prob = prob - GenerateState.args = args - pool = Pool(args.parallelism) - res = pool.map_async(generate_case, range(len(prob.generators._testcases))) - while not res.ready(): - # Use async polling for better KeyboardInterrupt handling - res.wait(1) - res = res.get() - return [ sum( r[tp] for r in res ) for tp in range(3) ] - - -def main(): - args = argparser().parse_args() - args.parts = ['generators'] - if args.clean_all: - args.clean = True - if not args.clean: - args.generate = True - args.compile_generators = args.generate - if args.parallelism is None: - args.parallelism = cpu_count() - initialize_logging(args) - - total_errors = 0 - for problemdir in args.problemdir: - print('Loading problem %s' % os.path.basename(os.path.realpath(problemdir))) - with Problem(problemdir) as prob: - prob.check(args) - errors = ProblemAspect.errors - warnings = ProblemAspect.warnings - - if prob.shortname is None: - # Skip invalid problem - continue - - def p(x): - return '' if x == 1 else 's' - - status = '' - if args.clean: - cnt, clean_errors, clean_warnings = clean(prob, args) - status += '%d case%s cleaned, ' % (cnt, p(cnt)) - errors += clean_errors - warnings += clean_warnings - if args.generate: - cnt, gen_errors, gen_warnings = generate(prob, args) - status += '%d case%s generated, ' % (cnt, p(cnt)) - errors += gen_errors - warnings += gen_warnings - - print("%s processed: %s%d error%s, %d warning%s" % (prob.shortname, status, errors, p(errors), warnings, p(warnings))) - total_errors += errors - - sys.exit(1 if total_errors > 0 else 0) - -if __name__ == '__main__': - main() diff --git a/problemtools/languages.py b/problemtools/languages.py index 8c0c72c2..93941205 100644 --- a/problemtools/languages.py +++ b/problemtools/languages.py @@ -2,17 +2,19 @@ This module contains functionality for reading and using configuration of programming languages. """ + import fnmatch import re import string +from pathlib import Path from . import config + class LanguageConfigError(Exception): """Exception class for errors in language configuration.""" - pass - + pass class Language(object): @@ -42,7 +44,6 @@ def __init__(self, lang_id, lang_spec): self.run = None self.update(lang_spec) - def get_source_files(self, file_list): """Given a list of files, determine which ones would be considered source files for the language. @@ -50,12 +51,14 @@ def get_source_files(self, file_list): Args: file_list (list of str): list of file names """ - return [file_name for file_name in file_list - if (any(fnmatch.fnmatch(file_name, glob) - for glob in self.files) - and - self.__matches_shebang(file_name))] - + return [ + file_name + for file_name in file_list + if ( + any(fnmatch.fnmatch(file_name, glob) for glob in self.files) # type: ignore[union-attr] + and self.__matches_shebang(file_name) + ) + ] def update(self, values): """Update a language specification with new values. @@ -66,23 +69,17 @@ def update(self, values): """ # Check that all provided values are known keys - for unknown in set(values)-set(Language.__KEYS): - raise LanguageConfigError( - 'Unknown key "%s" specified for language %s' - % (unknown, self.lang_id)) + for unknown in set(values) - set(Language.__KEYS): + raise LanguageConfigError('Unknown key "%s" specified for language %s' % (unknown, self.lang_id)) - for (key, value) in values.items(): + for key, value in values.items(): # Check type if key == 'priority': if not isinstance(value, int): - raise LanguageConfigError( - 'Language %s: priority must be integer but is %s.' - % (self.lang_id, type(value))) + raise LanguageConfigError('Language %s: priority must be integer but is %s.' % (self.lang_id, type(value))) else: if not isinstance(value, str): - raise LanguageConfigError( - 'Language %s: %s must be string but is %s.' - % (self.lang_id, key, type(value))) + raise LanguageConfigError('Language %s: %s must be string but is %s.' % (self.lang_id, key, type(value))) # Save the value if key == 'shebang': @@ -97,7 +94,6 @@ def update(self, values): self.__check() - def __check(self): """Check that the language specification is valid (all mandatory fields provided, all metavariables used in compile/run @@ -105,46 +101,33 @@ def __check(self): """ # Check that all mandatory fields are provided if self.name is None: - raise LanguageConfigError( - 'Language %s has no name' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no name') if self.priority is None: - raise LanguageConfigError( - 'Language %s has no priority' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no priority') if self.files is None: - raise LanguageConfigError( - - 'Language %s has no files glob' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no files glob') if self.run is None: - raise LanguageConfigError( - 'Language %s has no run command' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no run command') # Check that all variables appearing are valid variables = Language.__variables_in_command(self.run) if self.compile is not None: variables = variables | Language.__variables_in_command(self.compile) for unknown in variables - set(Language.__VARIABLES): - raise LanguageConfigError( - 'Unknown variable "{%s}" used for language %s' - % (unknown, self.lang_id)) + raise LanguageConfigError('Unknown variable "{%s}" used for language %s' % (unknown, self.lang_id)) # Check for uniquely defined entry point entry = variables & set(['binary', 'mainfile', 'mainclass', 'Mainclass']) if len(entry) == 0: - raise LanguageConfigError( - 'No entry point variable used for language %s' % self.lang_id) + raise LanguageConfigError('No entry point variable used for language %s' % self.lang_id) if len(entry) > 1: - raise LanguageConfigError( - 'More than one entry point type variable used for language %s' - % self.lang_id) - + raise LanguageConfigError('More than one entry point type variable used for language %s' % self.lang_id) @staticmethod def __variables_in_command(cmd): """List all meta-variables appearing in a string.""" formatter = string.Formatter() - return set(field for _, field, _, _ in formatter.parse(cmd) - if field is not None) - + return set(field for _, field, _, _ in formatter.parse(cmd) if field is not None) def __matches_shebang(self, filename): """Check if a file matches the shebang rule for the language.""" @@ -155,10 +138,6 @@ def __matches_shebang(self, filename): return self.shebang.search(shebang_line) is not None - - - - class Languages(object): """A set of languages.""" @@ -174,7 +153,6 @@ def __init__(self, data=None): if data is not None: self.update(data) - def detect_language(self, file_list): """Auto-detect language for a set of files. @@ -186,7 +164,7 @@ def detect_language(self, file_list): list of files did not match any language in the set. """ result = None - src = [] + src: list[str] = [] prio = 1e99 for lang in self.languages.values(): lang_src = lang.get_source_files(file_list) @@ -198,9 +176,7 @@ def detect_language(self, file_list): def get(self, lang_id): if not isinstance(lang_id, str): - raise LanguageConfigError( - 'Config file error: language IDs must be strings, but %s is %s.' - % (lang_id, type(lang_id))) + raise LanguageConfigError('Config file error: language IDs must be strings, but %s is %s.' % (lang_id, type(lang_id))) return self.languages.get(lang_id, None) def update(self, data): @@ -213,21 +189,19 @@ def update(self, data): for that language will be overridden and updated. """ if not isinstance(data, dict): - raise LanguageConfigError( - 'Config file error: content must be a dictionary, but is %s.' - % (type(data))) + raise LanguageConfigError('Config file error: content must be a dictionary, but is %s.' % (type(data))) - for (lang_id, lang_spec) in data.items(): + for lang_id, lang_spec in data.items(): if not isinstance(lang_id, str): raise LanguageConfigError( - 'Config file error: language IDs must be strings, but %s is %s.' - % (lang_id, type(lang_id))) + 'Config file error: language IDs must be strings, but %s is %s.' % (lang_id, type(lang_id)) + ) if not isinstance(lang_spec, (dict, Language)): raise LanguageConfigError( 'Config file error: language spec must be a dictionary, but spec of language %s is %s.' - % (lang_id, type(lang_spec))) - + % (lang_id, type(lang_spec)) + ) if isinstance(lang_spec, Language): self.languages[lang_id] = lang_spec @@ -236,19 +210,19 @@ def update(self, data): else: self.languages[lang_id].update(lang_spec) - priorities = {} - for (lang_id, lang) in self.languages.items(): + priorities: dict[int, Language] = {} + for lang_id, lang in self.languages.items(): if lang.priority in priorities: raise LanguageConfigError( - 'Languages %s and %s both have priority %d.' - % (lang_id, priorities[lang.priority], lang.priority)) + 'Languages %s and %s both have priority %d.' % (lang_id, priorities[lang.priority], lang.priority) + ) priorities[lang.priority] = lang_id -def load_language_config(): +def load_language_config(probdir_parent: Path) -> Languages: """Load language configuration. Returns: Languages object for the set of languages. """ - return Languages(config.load_config('languages.yaml')) + return Languages(config.load_config('languages.yaml', [probdir_parent])) diff --git a/problemtools/md2html.py b/problemtools/md2html.py new file mode 100644 index 00000000..445eca0f --- /dev/null +++ b/problemtools/md2html.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +import argparse +import hashlib +import html +import os +from pathlib import Path +import re +import shutil +import string +import subprocess + +import nh3 + +from . import statement_util + + +def convert(problem_root: Path, options: argparse.Namespace, statement_file: Path) -> bool: + """Convert a Markdown statement to HTML. Writes output to current working directory. + + Args: + problem: path to problem directory + options: command-line arguments. See problem2html.py + """ + destfile = string.Template(options.destfile).safe_substitute(problem=problem_root.name) + imgbasedir = string.Template(options.imgbasedir).safe_substitute(problem=problem_root.name) + + command = ['pandoc', str(statement_file), '-t', 'html', '--mathjax'] + statement_html = subprocess.run(command, capture_output=True, text=True, shell=False, check=True).stdout + + statement_html = sanitize_html(statement_file.parent, statement_html, imgbasedir) + + templatepaths = [ + os.path.join(os.path.dirname(__file__), 'templates/markdown_html'), + '/usr/lib/problemtools/templates/markdown_html', + ] + templatepath = next( + (p for p in templatepaths if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'default-layout.html'))), None + ) + + if templatepath is None: + raise FileNotFoundError('Could not find directory with markdown templates') + + with open(Path(templatepath) / 'default-layout.html', 'r', encoding='utf-8') as template_file: + template = template_file.read() + + problem_name = statement_util.get_yaml_problem_name(problem_root, options.language) + substitution_params = { + 'statement_html': statement_html, + 'language': options.language, + 'title': html.escape(problem_name) if problem_name else 'Missing problem name', + 'problemid': html.escape(problem_root.name), + } + + statement_html = template % substitution_params + + samples = statement_util.format_samples(problem_root) + # Insert samples at {{nextsample}} and {{remainingsamples}} + statement_html, remaining_samples = statement_util.inject_samples(statement_html, samples) + + # Insert the remaining samples at the bottom + # However, footnotes should be below samples + sample_insertion_position = statement_util.find_footnotes(statement_html) + if sample_insertion_position is None: + # No footnotes, so insert at the end + sample_insertion_position = statement_html.rfind('') + statement_html = ( + statement_html[:sample_insertion_position] + ''.join(remaining_samples) + statement_html[sample_insertion_position:] + ) + + with open(destfile, 'w', encoding='utf-8', errors='xmlcharrefreplace') as output_file: + output_file.write(statement_html) + + if options.css: + shutil.copyfile(os.path.join(templatepath, 'problem.css'), 'problem.css') + + return True + + +def sanitize_html(statement_dir: Path, statement_html: str, imgbasedir: str) -> str: + # Allow footnote ids (the anchor points you jump to) + def is_fn_id(s): + pattern_id_top = r'^fn\d+$' + pattern_id_bottom = r'^fnref\d+$' + return bool(re.fullmatch(pattern_id_top, s)) or bool(re.fullmatch(pattern_id_bottom, s)) + + allowed_classes = ('sample', 'problemheader', 'problembody', 'sampleinteractionwrite', 'sampleinteractionread') + + # Annoying: nh3 will ignore exceptions in attribute_filter + image_fail_reason: list[Exception] = [] + + def attribute_filter(tag, attribute, value): + if attribute == 'class' and value in allowed_classes: + return value + # Never versions of Pandoc will give class="footnotes footnotes-end-of-document" + # We don't want to blindly allow any class with footnotes in it, so only allow footnotes + if attribute == 'class' and 'footnotes' in value: + return 'footnotes' + if tag == 'a' and attribute == 'href': + return value + if tag in ('li', 'a') and attribute == 'id' and is_fn_id(value): + return value + if tag == 'img' and attribute == 'src': + try: + statement_util.assert_image_is_valid(statement_dir, value) + except Exception as e: + nonlocal image_fail_reason + image_fail_reason.append(e) + return None + return copy_image(statement_dir, value, imgbasedir) + return None + + statement_html = nh3.clean( + statement_html, + link_rel='noopener nofollow noreferrer', + attribute_filter=attribute_filter, + tags=nh3.ALLOWED_TAGS | {'img', 'a', 'section'}, + attributes={ + 'table': {'class'}, + 'aside': {'class'}, + 'div': {'class'}, + 'section': {'class'}, + 'img': {'src'}, + 'a': {'href', 'id'}, + 'li': {'id'}, + }, + ) + + if image_fail_reason: + # We don't have a great way to emit multiple errors from here, so just re-raise the first error + raise image_fail_reason[0] + + return statement_html + + +def copy_image(statement_dir: Path, img_src: str, imgbasedir: str) -> str: + """Copy image to working directory (with new filename) and returns the new filename + + Args: + statement_dir: the directory with problem statement files + img_src: the image source as in the Markdown statement + """ + + # We rename to sha256 of contents, and preserve the suffix. This flattens + # the directory structure to a single folders in a simple way. + with open(statement_dir / img_src, 'rb') as f: + filename = hashlib.file_digest(f, 'sha256').hexdigest() + Path(img_src).suffix + + if not os.path.isfile(filename): # check if already copied + shutil.copyfile(statement_dir / img_src, filename) + return imgbasedir + filename diff --git a/problemtools/metadata.py b/problemtools/metadata.py new file mode 100644 index 00000000..d01f7f10 --- /dev/null +++ b/problemtools/metadata.py @@ -0,0 +1,375 @@ +import copy +import datetime +import re +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import Any, Literal, Self, Type, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field +import yaml + +from . import config +from . import statement_util +from .formatversion import FormatVersion + + +class ProblemType(StrEnum): + PASS_FAIL = 'pass-fail' + SCORING = 'scoring' + MULTI_PASS = 'multi-pass' + INTERACTIVE = 'interactive' + SUBMIT_ANSWER = 'submit-answer' + + +class License(StrEnum): + UNKNOWN = 'unknown' + PUBLIC_DOMAIN = 'public domain' + CC0 = 'cc0' + CC_BY = 'cc by' + CC_BY_SA = 'cc by-sa' + EDUCATIONAL = 'educational' + PERMISSION = 'permission' + + +@dataclass +class Person: + name: str + email: str | None = None + orcid: str | None = None + kattis: str | None = None + + @classmethod + def from_string(cls: Type[Self], s: str) -> Self: + match = re.match(r'^(.*?)\s+<(.*)>$', s.strip()) + if match: + return cls(name=match.group(1), email=match.group(2)) + return cls(name=s) + + +@dataclass +class Source: + name: str + url: str | None = None + + +@dataclass +class TimeMultipliers: + ac_to_time_limit: float = 2.0 + time_limit_to_tle: float = 1.5 + + +@dataclass +class Limits: + memory: int + output: int + code: int + compilation_time: int + compilation_memory: int + validation_time: int + validation_memory: int + validation_output: int + time_multipliers: TimeMultipliers = field(default_factory=TimeMultipliers) + time_limit: float | None = None + time_resolution: float = 1.0 + validation_passes: int = 2 + + +@dataclass +class Credits: + """ + Credits format where all persons have been converted to Person objects. + For use in our internal representation. + """ + + authors: list[Person] = field(default_factory=list) + contributors: list[Person] = field(default_factory=list) + testers: list[Person] = field(default_factory=list) + translators: dict[str, list[Person]] = field(default_factory=dict) + packagers: list[Person] = field(default_factory=list) + acknowledgements: list[Person] = field(default_factory=list) + + +@dataclass +class InputCredits: + """ + A more permissive dataclass for credits, as the input in 2023-07 looks. + For use when validating input. + """ + + # Type in the input format is messy + PersonOrPersons = Union[str | list[Union[Person, str]]] + + authors: PersonOrPersons = field(default_factory=list) + contributors: PersonOrPersons = field(default_factory=list) + testers: PersonOrPersons = field(default_factory=list) + translators: dict[str, PersonOrPersons] = field(default_factory=dict) + packagers: PersonOrPersons = field(default_factory=list) + acknowledgements: PersonOrPersons = field(default_factory=list) + + +class Metadata2023_07(BaseModel): + """ + The metadata for a problem as input in version 2023-07-draft. + """ + + problem_format_version: str + name: dict[str, str] | str + uuid: UUID | None = None # UUID *is* mandatory, but we deal with that in verifyproblem for better UX + type: list[ProblemType] | ProblemType = ProblemType.PASS_FAIL + version: str | None = None + credits: dict | str | None = None + source: list[Union[str, Source]] | Source | str = [] + license: License = License.UNKNOWN + rights_owner: str | None = None + embargo_until: datetime.datetime | None = None + limits: Limits + keywords: list[str] = [] + languages: list[str] | Literal['all'] = 'all' + allow_file_writing: bool = True + constants: dict[str, int | float | str] = {} + + model_config = ConfigDict(extra='forbid') + + +@dataclass +class LegacyGrading: + objective: Literal['max', 'min'] = 'max' + show_test_data_groups: bool = False + # These 3 fields predate the version called "legacy" + accept_score: float | None = None + reject_score: float | None = None + range: str | None = None + on_reject: Literal['first_error', 'worst_error', 'grade'] | None = None + + +@dataclass +class LegacyLimits: + memory: int + output: int + code: int + compilation_time: int + compilation_memory: int + validation_time: int + validation_memory: int + validation_output: int + time_multiplier: float = 5.0 + time_safety_margin: float = 2.0 + + +class MetadataLegacy(BaseModel): + """ + The metadata for a problem as input in version legacy (plus a few fields + which pre-date the version called legacy). + """ + + problem_format_version: FormatVersion = FormatVersion.LEGACY + type: Literal['pass-fail'] | Literal['scoring'] = 'pass-fail' + name: str | None = None + uuid: UUID | None = None + author: str | None = None + source: str | None = None + source_url: str | None = None + license: License = License.UNKNOWN + rights_owner: str | None = None + limits: LegacyLimits + validation: str = 'default' + validator_flags: str = '' + grading: LegacyGrading = LegacyGrading() + keywords: str = '' + + model_config = ConfigDict(extra='forbid') + + +class Metadata(BaseModel): + """ + The metadata for a problem, as used internally in problemtools. Closely + follows the 2023-07-draft version, but is more fully parsed, and adds + a few legacy fields to represent information not in 2023-07. + + Metadata serializes to a valid 2023-07-draft configuration. + """ + + problem_format_version: FormatVersion + type: list[ProblemType] + name: dict[str, str] + uuid: UUID | None + version: str | None + credits: Credits + source: list[Source] + license: License + rights_owner: str | None + embargo_until: datetime.datetime | None + limits: Limits + keywords: list[str] + languages: list[str] | Literal['all'] + allow_file_writing: bool + constants: dict + legacy_grading: LegacyGrading = Field(default_factory=LegacyGrading, exclude=True) + legacy_validation: str = Field(default='', exclude=True) + legacy_validator_flags: str = Field(default='', exclude=True) + legacy_custom_score: bool = Field(default=False, exclude=True) # True iff legacy_validation is custom and score. + + model_config = ConfigDict(extra='forbid') + + def is_pass_fail(self) -> bool: + return not self.is_scoring() + + def is_scoring(self) -> bool: + return ProblemType.SCORING in self.type + + def is_custom_score_allowed(self) -> bool: + match self.problem_format_version: + case FormatVersion.LEGACY: + return self.legacy_custom_score + case FormatVersion.V_2023_07: + return self.is_scoring() + + def is_custom_score_mandatory(self) -> bool: + match self.problem_format_version: + case FormatVersion.LEGACY: + return self.legacy_custom_score + case FormatVersion.V_2023_07: + return False + + def is_interactive(self) -> bool: + return ProblemType.INTERACTIVE in self.type + + def is_multi_pass(self) -> bool: + return ProblemType.MULTI_PASS in self.type + + def is_submit_answer(self) -> bool: + return ProblemType.SUBMIT_ANSWER in self.type + + @classmethod + def from_legacy(cls: Type[Self], legacy: MetadataLegacy, names_from_statements: dict[str, str]) -> Self: + metadata = legacy.model_dump() + metadata['type'] = [metadata['type']] + # Support for *ancient* problems where names_from_statements is empty + if names_from_statements: + metadata['name'] = names_from_statements + elif metadata['name']: + metadata['name'] = {'en': metadata['name']} + else: + metadata['name'] = {} + metadata['version'] = None + + def parse_author_field(author: str) -> list[Person]: + authors = re.split(r',\s*|\s+and\s+|\s+&\s+', author) + authors = [x.strip(' \t\r\n') for x in authors] + authors = [x for x in authors if len(x) > 0] + return [Person.from_string(author) for author in authors] + + metadata['credits'] = {} + if metadata['author'] is not None: + metadata['credits']['authors'] = parse_author_field(metadata['author']) + del metadata['author'] + metadata['source'] = [] if metadata['source'] is None else [Source(metadata['source'], metadata['source_url'])] + del metadata['source_url'] + metadata['embargo_until'] = None + metadata['limits']['time_multipliers'] = { + 'ac_to_time_limit': metadata['limits']['time_multiplier'], + 'time_limit_to_tle': metadata['limits']['time_safety_margin'], + } + del metadata['limits']['time_multiplier'] + del metadata['limits']['time_safety_margin'] + metadata['keywords'] = metadata['keywords'].split() + metadata['languages'] = 'all' + metadata['allow_file_writing'] = True + metadata['constants'] = {} + + # The interactive flag from validation now lives in type, copy it over. + validation = metadata['validation'].split() + if validation[0] == 'custom': + if 'interactive' in validation[1:]: + metadata['type'].append('interactive') + if 'score' in validation[1:]: + metadata['legacy_custom_score'] = True + # Copy over the legacy info that does not fit cleanly + for key in 'grading', 'validator_flags', 'validation': + metadata[f'legacy_{key}'] = metadata[key] + del metadata[key] + return cls.model_validate(metadata) + + @classmethod + def from_2023_07(cls: Type[Self], md2023_07: Metadata2023_07) -> Self: + metadata = md2023_07.model_dump() + metadata['type'] = [metadata['type']] if isinstance(metadata['type'], str) else metadata['type'] + metadata['name'] = {'en': metadata['name']} if isinstance(metadata['name'], str) else metadata['name'] + + def parse_source(source: str | Source) -> Source: + return Source(name=source, url=None) if isinstance(source, str) else source + + # Convenience function to deal with the fact that lists of persons/sources are + # either a string, or a list of strings or dicts (if dicts, pydantic + # already parsed those for us). + def parse_list(callback, lst: str | list) -> list: + if isinstance(lst, str): + return [callback(lst)] + return list(map(callback, lst)) + + metadata['source'] = parse_list(parse_source, metadata['source']) + + def parse_person(person: str | Person) -> Person: + return Person.from_string(person) if isinstance(person, str) else person + + if metadata['credits'] is None: + metadata['credits'] = {} + elif isinstance(metadata['credits'], str): + metadata['credits'] = {'authors': [parse_person(metadata['credits'])]} + else: + for key in metadata['credits']: + if key == 'translators': # special case, we nest deeper here + for lang in metadata['credits'][key]: + metadata['credits'][key][lang] = parse_list(parse_person, metadata['credits'][key][lang]) + else: + metadata['credits'][key] = parse_list(parse_person, metadata['credits'][key]) + return cls.model_validate(metadata) + + +def parse_metadata( + version: FormatVersion, + problem_yaml_data: dict[str, Any], + names_from_statements: dict[str, str] | None = None, +) -> Metadata: + """ + Parses a data structure from problem.yaml into a Metadata model + :raises pydantic.ValidationError: We intentionally leak pydantic's exception on errors, as it's well designed + """ + + # We need to mix in the system default config values before doing model validation + data = copy.deepcopy(problem_yaml_data) + # Check if the user has done something silly like making limits a string. If so, we + # don't merge in anything, and let pydantic complain later. + if isinstance(data.get('limits', {}), dict): + system_defaults = config.load_config('problem.yaml') + data['limits'] = system_defaults['limits'] | data.get('limits', {}) + + if version is FormatVersion.LEGACY: + legacy_model = MetadataLegacy.model_validate(data) + return Metadata.from_legacy(legacy_model, names_from_statements or {}) + else: + assert version is FormatVersion.V_2023_07 + model_2023_07 = Metadata2023_07.model_validate(data) + return Metadata.from_2023_07(model_2023_07) + + +def load_metadata(problem_root: Path) -> tuple[Metadata, dict]: + """ + Loads metadata from a problem directory. + + Returns Metadata as well as the raw parsed yaml. The latter is likely only of use to verifyproblem. + Leaks exceptions, which is a bit of a mess. Unclear how to best deal with error handling. + """ + with (problem_root / 'problem.yaml').open() as f: + data = yaml.safe_load(f) + if data is None: # Loading empty yaml returns None + data = {} + + version = FormatVersion(data.get('problem_format_version', FormatVersion.LEGACY)) + if version is FormatVersion.LEGACY: + names_from_statements = statement_util.load_names_from_statements(problem_root, version) + else: + names_from_statements = None + return parse_metadata(version, data, names_from_statements), data diff --git a/problemtools/problem2html.py b/problemtools/problem2html.py index f4306d5d..918438f2 100644 --- a/problemtools/problem2html.py +++ b/problemtools/problem2html.py @@ -1,71 +1,30 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- -import re +import argparse import os.path +import re import string -import argparse -import logging import subprocess +import sys +from pathlib import Path -import plasTeX.TeX -import plasTeX.Logging - -from .ProblemPlasTeX import ProblemRenderer -from .ProblemPlasTeX import ProblemsetMacros -from . import template +from . import tex2html +from . import md2html +from . import statement_util +from .version import add_version_arg -def convert(args: list[str]|None = None) -> None: - options = parse_args(args) - problem = os.path.realpath(options.problem) +def convert(options: argparse.Namespace, force_statement_file: Path | None = None) -> None: + problem_root = Path(options.problem).resolve(strict=True) - problembase = os.path.splitext(os.path.basename(problem))[0] - destdir = string.Template(options.destdir).safe_substitute(problem=problembase) - destfile = string.Template(options.destfile).safe_substitute(problem=problembase) - imgbasedir = string.Template(options.imgbasedir).safe_substitute(problem=problembase) - - if options.quiet: - plasTeX.Logging.disableLogging() + if force_statement_file: # Used by verifyproblem to test rendering even if there are multiple statements in a language + statement_file = force_statement_file else: - plasTeX.Logging.getLogger().setLevel(getattr(logging, options.loglevel.upper())) - plasTeX.Logging.getLogger('status').setLevel(getattr(logging, options.loglevel.upper())) - - texfile = problem - # Set up template if necessary - with template.Template(problem, language=options.language) as templ: - texfile = open(templ.get_file_name(), 'r') - - origcwd = os.getcwd() - - # Setup parser and renderer etc + statement_file = statement_util.find_statement(problem_root, options.language) - # plasTeX version 3 changed the name of this argument (and guarding against this - # by checking plasTeX.__version__ fails on plastex v3.0 which failed to update - # __version__) - try: - tex = plasTeX.TeX.TeX(myfile=texfile) - except Exception: - tex = plasTeX.TeX.TeX(file=texfile) - - ProblemsetMacros.init(tex) - - tex.ownerDocument.config['general']['copy-theme-extras'] = options.css - if not options.headers: - tex.ownerDocument.userdata['noheaders'] = True - tex.ownerDocument.config['files']['filename'] = destfile - tex.ownerDocument.config['images']['filenames'] = 'img-$num(4)' - tex.ownerDocument.config['images']['enabled'] = False - tex.ownerDocument.config['images']['imager'] = 'none' - tex.ownerDocument.config['images']['base-url'] = imgbasedir - # tell plasTeX where to search for problemtools' built-in packages - tex.ownerDocument.config['general']['packages-dirs'] = [os.path.join(os.path.dirname(__file__), 'ProblemPlasTeX')] - - renderer = ProblemRenderer() - - if not options.quiet: - print('Parsing TeX source...') - doc = tex.parse() - texfile.close() + destdir = string.Template(options.destdir).safe_substitute(problem=problem_root.name) + destfile = string.Template(options.destfile).safe_substitute(problem=problem_root.name) + origcwd = os.getcwd() # Go to destdir if destdir: @@ -76,12 +35,13 @@ def convert(args: list[str]|None = None) -> None: try: if not options.quiet: print('Rendering!') - renderer.render(doc) - - # Annoying: I have not figured out any way of stopping the plasTeX - # renderer from generating a .paux file - if os.path.isfile('.paux'): - os.remove('.paux') + match statement_file.suffix: + case '.md': + md2html.convert(problem_root, options, statement_file) + case '.tex': + tex2html.convert(problem_root, options, statement_file) + case _: + raise NotImplementedError('Unsupported file type, expected md or tex: {statement_file.name}') if options.tidy: with open(os.devnull, 'w') as devnull: @@ -97,9 +57,9 @@ def convert(args: list[str]|None = None) -> None: for f in files: file_size_kib = os.stat(os.path.join(path, f)).st_size // 1024 if file_size_kib > 1024: - print(f"WARNING: FILE {f} HAS SIZE {file_size_kib} KiB; CONSIDER REDUCING IT") + print(f'WARNING: FILE {f} HAS SIZE {file_size_kib} KiB; CONSIDER REDUCING IT') elif file_size_kib > 300: - print(f"Warning: file {f} has size {file_size_kib} KiB; consider reducing it") + print(f'Warning: file {f} has size {file_size_kib} KiB; consider reducing it') if options.bodyonly: content = open(destfile).read() @@ -111,30 +71,48 @@ def convert(args: list[str]|None = None) -> None: os.chdir(origcwd) -def parse_args(args: list[str]|None) -> argparse.Namespace: +def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-b', '--body-only', dest='bodyonly', action='store_true', help='only generate HTML body, no HTML headers', default=False) - parser.add_argument('-c', '--no-css', dest='css', action='store_false', help="don't copy CSS file to output directory", default=True) - parser.add_argument('-H', '--headers', dest='headers', action='store_false', help="don't generate problem headers (title, problem id, time limit)", default=True) - parser.add_argument('-m', '--messy', dest='tidy', action='store_false', help="don't run tidy to postprocess the HTML", default=True) - parser.add_argument('-d', '--dest-dir', dest='destdir', help="output directory", default='${problem}_html') - parser.add_argument('-f', '--dest-file', dest='destfile', help="output file name", default='index.html') - parser.add_argument('-l', '--language', dest='language', help='choose alternate language (2-letter code)', default=None) - parser.add_argument('-L', '--log-level', dest='loglevel', help='set log level (debug, info, warning, error, critical)', default='warning') - parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', help="quiet", default=False) + parser.add_argument( + '-b', '--body-only', dest='bodyonly', action='store_true', help='only generate HTML body, no HTML headers', default=False + ) + parser.add_argument( + '-c', '--no-css', dest='css', action='store_false', help="don't copy CSS file to output directory", default=True + ) + parser.add_argument( + '-H', + '--headers', + dest='headers', + action='store_false', + help="don't generate problem headers (title, problem id, time limit)", + default=True, + ) + parser.add_argument( + '-m', '--messy', dest='tidy', action='store_false', help="don't run tidy to postprocess the HTML", default=True + ) + parser.add_argument('-d', '--dest-dir', dest='destdir', help='output directory', default='${problem}_html') + parser.add_argument('-f', '--dest-file', dest='destfile', help='output file name', default='index.html') + parser.add_argument('-l', '--language', dest='language', help='choose language (2-letter code)', default='en') + parser.add_argument( + '-L', '--log-level', dest='loglevel', help='set log level (debug, info, warning, error, critical)', default='warning' + ) + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', help='quiet', default=False) + parser.add_argument('-i', '--imgbasedir', dest='imgbasedir', default='') parser.add_argument('problem', help='the problem to convert') + add_version_arg(parser) - if args is not None: - options = parser.parse_args(args) - else: - options = parser.parse_args() + return parser - options.imgbasedir = '' - return options def main() -> None: - convert() + parser = get_parser() + options = parser.parse_args() + try: + convert(options) + except Exception as e: + print(e) + sys.exit(1) if __name__ == '__main__': diff --git a/problemtools/problem2pdf.py b/problemtools/problem2pdf.py index 4c7a221b..1eaa38a6 100644 --- a/problemtools/problem2pdf.py +++ b/problemtools/problem2pdf.py @@ -1,36 +1,115 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- -import os.path +import argparse +import os +import re import shutil import string -import argparse import subprocess -from . import template - - -def convert(args: list[str]|None = None) -> bool: - options = parse_args(args) +import sys +import tempfile +from pathlib import Path - problem = os.path.realpath(options.problem) - problembase = os.path.splitext(os.path.basename(problem))[0] - destfile = string.Template(options.destfile).safe_substitute(problem=problembase) +from . import template +from . import statement_util +from .version import add_version_arg + + +def convert(options: argparse.Namespace, force_statement_file: Path | None = None) -> bool: + problem_root = Path(options.problem).resolve(strict=True) + + if force_statement_file: # Used by verifyproblem to test rendering even if there are multiple statements in a language + statement_file = force_statement_file + else: + statement_file = statement_util.find_statement(problem_root, options.language) + + match statement_file.suffix: + case '.md': + return md2pdf(options, statement_file) + case '.tex': + return latex2pdf(options, statement_file) + case _: + raise NotImplementedError('Unsupported file type, expected md or tex: {statement_file.name}') + + +def md2pdf(options: argparse.Namespace, statement_file: Path) -> bool: + """Renders a Markdown document to pdf. Uses pandoc md -> tex, then + reuses the normal tex -> pdf pipeline + """ + problem_root = Path(options.problem).resolve(strict=True) + + statement_util.assert_images_are_valid_md(statement_file) + + command = ['pandoc', str(statement_file), '-t', 'latex'] + try: + tex = subprocess.run(command, capture_output=True, text=True, shell=False, check=True).stdout + except subprocess.CalledProcessError as e: + print(f'Error compiling Markdown to pdf: {e.stderr}') + return False + + def format_latex_tables(latex_doc): + # Match table environments produced by pandoc + pattern = r""" + (\\begin\{longtable\}\[\]\{@\{\}) + ([a-z]) + ([a-z]*) + (@\{\}\}) + """ + + def replacer(match): + prefix = match.group(1)[:-3] + first_col = match.group(2) + other_cols = match.group(3) + suffix = match.group(4)[3:] + + # Combine columns with | separators + cols = [first_col] + list(other_cols) + return f'{prefix}|{"|".join(cols)}|{suffix} \\hline' + + return re.sub(pattern, replacer, latex_doc, flags=re.VERBOSE) + + # Add solid outline to tables + tex = format_latex_tables(tex) + tex = tex.replace(r'\toprule', '') + tex = tex.replace(r'\midrule', '') + tex = tex.replace(r'\endhead', '') + tex = tex.replace(r'\bottomrule', '') + tex = tex.replace(r'\tabularnewline', r'\\ \hline') + + # Fix sample inclusions commands + # Currently does not work, as normal problemtools tex -> pdf does not support it + tex = tex.replace(r'\{\{nextsample\}\}', r'\nextsample') + tex = tex.replace(r'\{\{remainingsamples\}\}', r'\remainingsamples') + + problem_name = statement_util.get_yaml_problem_name(problem_root, options.language) + tex = r'\problemname{' + problem_name + '}\n' + tex + with tempfile.NamedTemporaryFile(mode='w', suffix='.tex', dir=statement_file.parent) as temp_tex_file: + temp_tex_file.write(tex) + temp_tex_file.flush() + return latex2pdf(options, Path(temp_tex_file.name)) + + return False + + +def latex2pdf(options: argparse.Namespace, statement_file: Path) -> bool: + problem_root = Path(options.problem).resolve(strict=True) + destfile = string.Template(options.destfile).safe_substitute(problem=problem_root.name) - texfile = problem # Set up template if necessary - with template.Template(problem, language=options.language) as templ: + with template.Template(problem_root, statement_file, options.language) as templ: texfile = templ.get_file_name() origcwd = os.getcwd() - os.chdir(os.path.dirname(texfile)) - params = ['pdflatex', '-interaction=nonstopmode'] + os.chdir(texfile.parent) + params = ['lualatex', '--interaction=nonstopmode'] output = None if options.quiet: output = open(os.devnull, 'w') if options.nopdf: - params.append('-draftmode') + params.append('--draftmode') - params.append(texfile) + params.append(str(texfile.name)) status = subprocess.call(params, stdout=output) if status == 0: @@ -42,27 +121,57 @@ def convert(args: list[str]|None = None) -> bool: os.chdir(origcwd) if status == 0 and not options.nopdf: - shutil.move(os.path.splitext(texfile)[0] + '.pdf', destfile) - - return status == 0 - -def parse_args(args: list[str]|None) -> argparse.Namespace: + shutil.move(texfile.with_suffix('.pdf'), destfile) + + if status: + return False + + # We only sanitize if a PDF was created + if not options.nopdf: + try: + with tempfile.NamedTemporaryFile(suffix='.pdf') as f: + command = [ + 'gs', + '-q', + '-dBATCH', + '-sDEVICE=pdfwrite', + '-dNOPAUSE', + '-dCompatibilityLevel=1.7', + f'-sOutputFile={f.name}', + destfile, + ] + gs_status = subprocess.run(command, capture_output=True, text=True, shell=False, check=True) + if gs_status.returncode != 0: + return False + shutil.copy(f.name, destfile) + except subprocess.CalledProcessError as e: + print(f'Error sanitizing PDF: {e} {e.stderr}') + raise + + return True + + +def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-o', '--output', dest='destfile', help="output file name", default='${problem}.pdf') - parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', help="quiet", default=False) - parser.add_argument('-l', '--language', dest='language', help='choose alternate language (2-letter code)', default=None) + parser.add_argument('-o', '--output', dest='destfile', help='output file name', default='${problem}.pdf') + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', help='quiet', default=False) + parser.add_argument('-l', '--language', dest='language', help='choose language (2-letter code)', default='en') parser.add_argument('-n', '--no-pdf', dest='nopdf', action='store_true', help='run pdflatex in -draftmode', default=False) parser.add_argument('problem', help='the problem to convert') + add_version_arg(parser) - if args is not None: - return parser.parse_args(args) - - return parser.parse_args() + return parser def main() -> None: - convert() + parser = get_parser() + options = parser.parse_args() + try: + convert(options) + except Exception as e: + print(e) + sys.exit(1) if __name__ == '__main__': diff --git a/problemtools/tests/__init__.py b/problemtools/py.typed similarity index 100% rename from problemtools/tests/__init__.py rename to problemtools/py.typed diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index 79de6047..6f5791dc 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -1,22 +1,23 @@ """Package for managing execution of external programs in Kattis Problemtools. """ + import re import os from .buildrun import BuildRun from .checktestdata import Checktestdata -from .errors import ProgramError -from .executable import Executable +from .errors import ProgramError as ProgramError from .program import Program from .source import SourceCode from .viva import Viva -from .tools import get_tool_path, get_tool +from .tools import get_tool as get_tool, get_tool_path as get_tool_path from . import rutil -def find_programs(path, pattern='.*', language_config=None, work_dir=None, - include_dir=None, allow_validation_script=False): +def find_programs( + path, pattern='.*', language_config=None, work_dir=None, include_dir=None, allow_validation_script=False +) -> list[Program]: """Find all programs in a directory. Args: @@ -51,18 +52,19 @@ def find_programs(path, pattern='.*', language_config=None, work_dir=None, for name in sorted(os.listdir(path)): if re.match(pattern, name): fullpath = os.path.join(path, name) - run = get_program(fullpath, - language_config=language_config, - work_dir=work_dir, - include_dir=include_dir, - allow_validation_script=allow_validation_script) + run = get_program( + fullpath, + language_config=language_config, + work_dir=work_dir, + include_dir=include_dir, + allow_validation_script=allow_validation_script, + ) if run is not None: ret.append(run) return ret -def get_program(path, language_config=None, work_dir=None, include_dir=None, - allow_validation_script=False): +def get_program(path, language_config=None, work_dir=None, include_dir=None, allow_validation_script=False) -> Program | None: """Get a Program object for a program Args: @@ -102,13 +104,18 @@ def get_program(path, language_config=None, work_dir=None, include_dir=None, files = [path] else: build = os.path.join(path, 'build') - if os.path.isfile(build) and os.access(path, os.X_OK): + if os.path.isfile(build) and os.access(build, os.X_OK): return BuildRun(path, work_dir) files = rutil.list_files_recursive(path) if language_config is not None: lang = language_config.detect_language(files) if lang is not None: - return SourceCode(path, lang, - work_dir=work_dir, include_dir=include_dir) + if include_dir is not None: + lang_dir = os.path.join(include_dir, lang.lang_id) + build = os.path.join(lang_dir, 'build') + if os.path.isfile(build) and os.access(build, os.X_OK): + return BuildRun(path, work_dir=work_dir, include_dir=lang_dir) + + return SourceCode(path, lang, work_dir=work_dir, include_dir=include_dir) return None diff --git a/problemtools/run/buildrun.py b/problemtools/run/buildrun.py index 208527c0..404bcf5e 100644 --- a/problemtools/run/buildrun.py +++ b/problemtools/run/buildrun.py @@ -12,12 +12,13 @@ from .program import Program from . import rutil +log = logging.getLogger(__file__) + class BuildRun(Program): - """Class for build/run-script program. - """ + """Class for build/run-script program.""" - def __init__(self, path, work_dir=None): + def __init__(self, path, work_dir=None, include_dir=None): """Instantiate BuildRun object. Args: @@ -25,15 +26,11 @@ def __init__(self, path, work_dir=None): work_dir (str): name of temp directory in which to run the scripts (if None, will make new temp directory). """ + super().__init__() + if not os.path.isdir(path): raise ProgramError('%s is not a directory' % path) - build = os.path.join(path, 'build') - if not os.path.isfile(build): - raise ProgramError('%s does not have a build script' % path) - if not os.access(build, os.X_OK): - raise ProgramError('%s/build is not executable' % path) - if work_dir is None: work_dir = tempfile.mkdtemp() @@ -47,34 +44,35 @@ def __init__(self, path, work_dir=None): os.makedirs(self.path) rutil.add_files(path, self.path) + if include_dir is not None and os.path.isdir(include_dir): + rutil.add_files(include_dir, self.path) + # Check for existence of build script after copying include_dir, since that could contain the script + build = os.path.join(self.path, 'build') + if not os.path.isfile(build): + raise ProgramError('%s does not have a build script' % path) + if not os.access(build, os.X_OK): + raise ProgramError('%s/build is not executable' % path) - def __str__(self): + def __str__(self) -> str: """String representation""" return '%s/' % (self.path) - - _compile_result = None - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Run the build script.""" - if self._compile_result is not None: - return self._compile_result - with open(os.devnull, 'w') as devnull: status = subprocess.call(['./build'], stdout=devnull, stderr=devnull, cwd=self.path) run = os.path.join(self.path, 'run') if status: logging.debug('Build script failed (status %d) when compiling %s\n', status, self.name) - self._compile_result = (False, 'build script failed with exit code %d' % (status)) + return (False, 'build script failed with exit code %d' % (status)) elif not os.path.isfile(run) or not os.access(run, os.X_OK): - self._compile_result = (False, 'build script did not produce an executable called "run"') + return (False, 'build script did not produce an executable called "run"') else: - self._compile_result = (True, None) - return self._compile_result - + return (True, None) - def get_runcmd(self, cwd=None, memlim=None): + def get_runcmd(self, cwd=None, memlim=None) -> list[str]: """Run command for the program. Args: @@ -84,7 +82,6 @@ def get_runcmd(self, cwd=None, memlim=None): path = self.path if cwd is None else os.path.relpath(self.path, cwd) return [os.path.join(path, 'run')] - - def should_skip_memory_rlimit(self): + def should_skip_memory_rlimit(self) -> bool: """Ugly hack (see program.py for details).""" return True diff --git a/problemtools/run/checktestdata.py b/problemtools/run/checktestdata.py index 0fd4a4d7..939c1e30 100644 --- a/problemtools/run/checktestdata.py +++ b/problemtools/run/checktestdata.py @@ -9,8 +9,8 @@ class Checktestdata(Executable): - """Wrapper class for running Checktestdata scripts. - """ + """Wrapper class for running Checktestdata scripts.""" + _CTD_PATH = get_tool_path('checktestdata') def __init__(self, path): @@ -20,33 +20,26 @@ def __init__(self, path): path (str): path to .ctd source file """ if Checktestdata._CTD_PATH is None: - raise ProgramError( - 'Could not locate the Checktestdata program to run %s' % path) - super(Checktestdata, self).__init__(Checktestdata._CTD_PATH, - args=[path]) - + raise ProgramError('Could not locate the Checktestdata program to run %s' % path) + super().__init__(Checktestdata._CTD_PATH, args=[path]) - def __str__(self): + def __str__(self) -> str: """String representation""" return '%s' % (self.args[0]) - - _compile_result = None - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Syntax-check the Checktestdata script Returns: (False, None) if the Checktestdata script has syntax errors and (True, None) otherwise """ - if self._compile_result is None: - (status, _) = super(Checktestdata, self).run() - self._compile_result = ((os.WIFEXITED(status) and os.WEXITSTATUS(status) in [0, 1]), None) - return self._compile_result - + (status, _) = super().run() + return ((os.WIFEXITED(status) and os.WEXITSTATUS(status) in [0, 1]), None) - def run(self, infile='/dev/null', outfile='/dev/null', - errfile='/dev/null', args=None, timelim=1000): + def run( + self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', args=None, timelim=1000, memlim=1024, work_dir=None + ): """Run the Checktestdata script to validate an input file. Args: @@ -66,15 +59,13 @@ def run(self, infile='/dev/null', outfile='/dev/null', runtime (float): runtime of the Checktestdata process in seconds """ - (status, runtime) = super(Checktestdata, self).run(infile=infile, - outfile=outfile, - errfile=errfile, - args=args, - timelim=timelim) + (status, runtime) = super(Checktestdata, self).run( + infile=infile, outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir + ) # This is ugly, switches the accept exit status and our accept # exit status 42. if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: - return (42<<8, runtime) + return (42 << 8, runtime) if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: return (0, runtime) return (status, runtime) diff --git a/problemtools/run/errors.py b/problemtools/run/errors.py index b71bc1c9..4b8d1db9 100644 --- a/problemtools/run/errors.py +++ b/problemtools/run/errors.py @@ -2,7 +2,8 @@ Error handling. """ + class ProgramError(Exception): - """Base exception class for errors within the run package. - """ + """Base exception class for errors within the run package.""" + pass diff --git a/problemtools/run/executable.py b/problemtools/run/executable.py index b2538e22..8f3a67e7 100644 --- a/problemtools/run/executable.py +++ b/problemtools/run/executable.py @@ -1,13 +1,15 @@ """ Implementation of programs provided by an executable file. """ + import os from .program import Program from .errors import ProgramError + class Executable(Program): - """Class for executable files. - """ + """Class for executable files.""" + def __init__(self, path, args=None): """Instantiate executable object. @@ -17,6 +19,8 @@ def __init__(self, path, args=None): args: list of additional command line arguments that should be passed to the program every time it is executed. """ + super().__init__() + if not os.path.isfile(path) or not os.access(path, os.X_OK): raise ProgramError('%s is not an executable program' % path) self.path = path @@ -26,14 +30,8 @@ def __str__(self): """String representation""" return '%s' % (self.path) - def compile(self): - """Dummy implementation of the compile method -- nothing to check! - """ - return (True, None) - def get_runcmd(self, cwd=None, memlim=None): - """Command to run the program. - """ + """Command to run the program.""" return [self.path] + self.args def should_skip_memory_rlimit(self): diff --git a/problemtools/run/limit.py b/problemtools/run/limit.py index db062dbf..5e56f6cc 100644 --- a/problemtools/run/limit.py +++ b/problemtools/run/limit.py @@ -4,6 +4,7 @@ import resource + def check_limit_capabilities(logger): """Check if the problemtools process is run with appropriate capabilities to set rlimits, and if not, issue warnings. @@ -16,19 +17,21 @@ def check_limit_capabilities(logger): """ (_, cpu_hard) = resource.getrlimit(resource.RLIMIT_CPU) if cpu_hard != resource.RLIM_INFINITY: - logger.warning("Hard CPU rlimit of %d, runs involving higher CPU limits than this may behave incorrectly." - % cpu_hard) + logger.warning('Hard CPU rlimit of %d, runs involving higher CPU limits than this may behave incorrectly.' % cpu_hard) (_, stack_hard) = resource.getrlimit(resource.RLIMIT_STACK) if stack_hard != resource.RLIM_INFINITY: - logger.warning("Hard stack rlimit of %d so I can't set it to unlimited. I will keep it at %d. If you experience unexpected issues (in particular run-time errors) this may be the cause." - % (stack_hard, stack_hard)) + logger.warning( + "Hard stack rlimit of %d so I can't set it to unlimited. I will keep it at %d. If you experience unexpected issues (in particular run-time errors) this may be the cause." + % (stack_hard, stack_hard) + ) (_, mem_hard) = resource.getrlimit(resource.RLIMIT_AS) if mem_hard != resource.RLIM_INFINITY: - logger.warning("Hard memory rlimit of %.0f MB, runs involving a higher memory limit may behave incorrectly. If you experience unexpected issues (in particular run-time errors) this may be the cause." - % (mem_hard/1024.0/1024.0)) - + logger.warning( + 'Hard memory rlimit of %.0f MB, runs involving a higher memory limit may behave incorrectly. If you experience unexpected issues (in particular run-time errors) this may be the cause.' + % (mem_hard / 1024.0 / 1024.0) + ) def try_limit(limit, soft, hard): @@ -49,7 +52,6 @@ def try_limit(limit, soft, hard): resource.setrlimit(limit, (soft, hard)) - def __limit_less(lim1, lim2): """Helper function for comparing two rlimit values, handling "unlimited" correctly. diff --git a/problemtools/run/program.py b/problemtools/run/program.py index a86fb8e5..ae27c377 100644 --- a/problemtools/run/program.py +++ b/problemtools/run/program.py @@ -1,26 +1,40 @@ -"""Abstract base class for programs. -""" +"""Abstract base class for programs.""" + import os from . import limit import resource import signal import logging +import threading from .errors import ProgramError -class Program(object): - """Abstract base class for programs. - """ - runtime = 0 +from abc import ABC, abstractmethod + +log = logging.getLogger(__name__) + + +class Program(ABC): + """Abstract base class for programs.""" + + def __init__(self) -> None: + self.runtime = 0 + self._compile_lock = threading.Lock() + self._compile_result: tuple[bool, str | None] | None = None - def run(self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', - args=None, timelim=1000, memlim=1024, set_work_dir=False): + @abstractmethod + def get_runcmd(self, cwd=None, memlim=None) -> list[str]: + pass + + def run( + self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', args=None, timelim=1000, memlim=1024, work_dir=None + ): """Run the program. Args: infile (str): name of file to pass on stdin outfile (str): name of file to send stdout to - errfile (str): name of file to send stderr ro + errfile (str): name of file to send stderr to args (list of str): additional command-line arguments to pass to the program timelim (int): CPU time limit in seconds @@ -39,20 +53,29 @@ def run(self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', if self.should_skip_memory_rlimit(): memlim = None - status, runtime = self.__run_wait(runcmd + args, - infile, outfile, errfile, - timelim, memlim, self.path if set_work_dir else None) + status, runtime = self.__run_wait(runcmd + args, infile, outfile, errfile, timelim, memlim, work_dir) self.runtime = max(self.runtime, runtime) return status, runtime - def code_size(self): + def compile(self) -> tuple[bool, str | None]: + with self._compile_lock: + if self._compile_result is None: + self._compile_result = self.do_compile() + return self._compile_result + + def do_compile(self) -> tuple[bool, str | None]: + """Actually compile the program, if needed. Subclasses should override this method. + Do not call this manually -- use compile() instead.""" + return (True, None) + + def code_size(self) -> int: """Subclasses should override this method with the total size of the source code.""" return 0 - def should_skip_memory_rlimit(self): + def should_skip_memory_rlimit(self) -> bool: """Ugly workaround to accommodate Java -- the JVM will crash and burn if there is a memory rlimit applied and this will probably not change anytime soon [time of writing this: 2017-02-05], see @@ -67,11 +90,9 @@ def should_skip_memory_rlimit(self): """ return False - @staticmethod def __run_wait(argv, infile, outfile, errfile, timelim, memlim, working_directory=None): - logging.debug('run "%s < %s > %s 2> %s"', - ' '.join(argv), infile, outfile, errfile) + log.debug('run "%s < %s > %s 2> %s"', ' '.join(argv), infile, outfile, errfile) pid = os.fork() if pid == 0: # child try: @@ -84,39 +105,35 @@ def __run_wait(argv, infile, outfile, errfile, timelim, memlim, working_director # # This *shouldn't* cause any verdict changes given the setup for # interactive problems, but reset them anyway, for sanity. - if hasattr(signal, "SIGPIPE"): + if hasattr(signal, 'SIGPIPE'): signal.signal(signal.SIGPIPE, signal.SIG_DFL) - if hasattr(signal, "SIGXFZ"): + if hasattr(signal, 'SIGXFZ'): signal.signal(signal.SIGXFZ, signal.SIG_DFL) - if hasattr(signal, "SIGXFSZ"): + if hasattr(signal, 'SIGXFSZ'): signal.signal(signal.SIGXFSZ, signal.SIG_DFL) if timelim is not None: limit.try_limit(resource.RLIMIT_CPU, timelim, timelim + 1) if memlim is not None: limit.try_limit(resource.RLIMIT_AS, memlim * (1024**2), resource.RLIM_INFINITY) - limit.try_limit(resource.RLIMIT_STACK, - resource.RLIM_INFINITY, resource.RLIM_INFINITY) + limit.try_limit(resource.RLIMIT_STACK, resource.RLIM_INFINITY, resource.RLIM_INFINITY) Program.__setfd(0, infile, os.O_RDONLY) - Program.__setfd(1, outfile, - os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - Program.__setfd(2, errfile, - os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + Program.__setfd(1, outfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + Program.__setfd(2, errfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) if working_directory is not None: os.chdir(working_directory) os.execvp(argv[0], argv) except Exception as exc: - print("Oops. Fatal error in child process:") + print('Oops. Fatal error in child process:') print(exc) os.kill(os.getpid(), signal.SIGTERM) # Unreachable - logging.error("Unreachable part of run_wait reached") + log.error('Unreachable part of run_wait reached') os.kill(os.getpid(), signal.SIGTERM) (pid, status, rusage) = os.wait4(pid, 0) return status, rusage.ru_utime + rusage.ru_stime - @staticmethod def __setfd(fd, filename, flag): tmpfd = os.open(filename, flag) diff --git a/problemtools/run/rutil.py b/problemtools/run/rutil.py index a3bafd9c..62df8fbe 100644 --- a/problemtools/run/rutil.py +++ b/problemtools/run/rutil.py @@ -1,11 +1,12 @@ -"""Some utility functions for the run module. -""" +"""Some utility functions for the run module.""" + import errno import os import shutil from .errors import ProgramError + def add_files(src, dstdir): """Copy src to dstdir. @@ -36,8 +37,7 @@ def add_files(src, dstdir): except IOError as exc: # FIXME why is this specific error special-cased if exc.errno == errno.ENOENT: - raise ProgramError( - 'File not found when copying program:\n %s' % exc.filename) + raise ProgramError('File not found when copying program:\n %s' % exc.filename) raise @@ -49,6 +49,6 @@ def list_files_recursive(root): directory and its subdirectories. """ ret = [] - for (path, _, files) in os.walk(root): + for path, _, files in os.walk(root): ret.extend([os.path.join(root, path, filename) for filename in files]) return ret diff --git a/problemtools/run/source.py b/problemtools/run/source.py index a7724bda..514c6cf7 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -1,6 +1,7 @@ """ Implementation of programs provided by source code. """ + import re import os import shlex @@ -12,9 +13,12 @@ from .program import Program from . import rutil +log = logging.getLogger(__name__) + + class SourceCode(Program): - """Class representing a program provided by source code. - """ + """Class representing a program provided by source code.""" + def __init__(self, path, language, work_dir=None, include_dir=None): """Instantiate SourceCode object @@ -36,6 +40,7 @@ def __init__(self, path, language, work_dir=None, include_dir=None): then the files in include_dir// will be copied into the work_dir along with the source file(s). """ + super().__init__() if path[-1] == '/': path = path[:-1] @@ -58,16 +63,11 @@ def __init__(self, path, language, work_dir=None, include_dir=None): if os.path.isdir(include_dir): rutil.add_files(include_dir, self.path) - self.src = sorted(self.language.get_source_files( - rutil.list_files_recursive(self.path) - )) + self.src = sorted(self.language.get_source_files(rutil.list_files_recursive(self.path))) if len(self.src) == 0: - raise ProgramError('No source files found for language %s in %s' - % (self.language.lang_id, self.name)) + raise ProgramError('No source files found for language %s in %s' % (self.language.lang_id, self.name)) - self.mainfile = next((x for x in self.src - if re.match(r'^main\.', os.path.basename(x), - re.IGNORECASE)), None) + self.mainfile = next((x for x in self.src if re.match(r'^main\.', os.path.basename(x), re.IGNORECASE)), None) if self.mainfile is None: self.mainfile = self.src[0] @@ -76,25 +76,17 @@ def __init__(self, path, language, work_dir=None, include_dir=None): self.binary = os.path.join(self.path, 'run') - - def code_size(self): + def code_size(self) -> int: return sum(os.path.getsize(x) for x in self.src) - - _compile_result = None - - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Compile the source code. Returns tuple: (True, None) if compilation succeeded (False, errmsg) otherwise """ - if self._compile_result is not None: - return self._compile_result - if self.language.compile is None: - self._compile_result = (True, None) return (True, None) command = self.get_compilecmd() @@ -103,21 +95,17 @@ def compile(self): if not os.path.isfile(compiler) or not os.access(compiler, os.X_OK): return (False, '%s does not seem to be installed, expected to find compiler at %s' % (self.language.name, compiler)) - logging.debug('compile command: %s', command) + log.debug('compile command: %s', command) try: subprocess.check_output(command, stderr=subprocess.STDOUT) - self._compile_result = (True, None) + return (True, None) except subprocess.CalledProcessError as err: - self._compile_result = (False, err.output.decode('utf8', 'replace')) - - return self._compile_result + return (False, err.output.decode('utf8', 'replace')) - - def get_compilecmd(self): + def get_compilecmd(self) -> list[str]: return shlex.split(self.language.compile.format(**self.__get_substitution())) - def get_runcmd(self, cwd=None, memlim=1024): """Run command for the program. @@ -136,17 +124,14 @@ def get_runcmd(self, cwd=None, memlim=1024): subs['mainfile'] = os.path.relpath(subs['mainfile'], cwd) return shlex.split(self.language.run.format(**subs)) - - def should_skip_memory_rlimit(self): + def should_skip_memory_rlimit(self) -> bool: """Ugly hack (see program.py for details).""" return self.language.name in ['Java', 'Scala', 'Kotlin', 'Common Lisp'] - - def __str__(self): + def __str__(self) -> str: """String representation""" return '%s (%s)' % (self.name, self.language.name) - def __get_substitution(self, memlim=1024): return { 'path': self.path, @@ -155,5 +140,5 @@ def __get_substitution(self, memlim=1024): 'mainfile': self.mainfile, 'mainclass': self.mainclass, 'Mainclass': self.Mainclass, - 'binary': self.binary + 'binary': self.binary, } diff --git a/problemtools/run/tools.py b/problemtools/run/tools.py index 78408cc2..23c30614 100644 --- a/problemtools/run/tools.py +++ b/problemtools/run/tools.py @@ -1,6 +1,7 @@ import os from .executable import Executable + def get_tool_path(name): """Find the path to one of problemtools' external tools. @@ -11,11 +12,12 @@ def get_tool_path(name): Returns: str, path to the tool, or None if the tool was not found. """ - return __locate_executable([os.path.join(os.path.dirname(__file__), - '..', 'support', name), - os.path.join(os.path.dirname(__file__), - '..', '..', 'support', - os.path.splitext(name)[0], name)]) + return __locate_executable( + [ + os.path.join(os.path.dirname(__file__), '..', 'support', name), + os.path.join(os.path.dirname(__file__), '..', '..', 'support', os.path.splitext(name)[0], name), + ] + ) def get_tool(name): @@ -43,5 +45,4 @@ def __locate_executable(candidate_paths): str, first entry of candidate_paths that is an executable file, or None if no such entry. """ - return next((p for p in candidate_paths - if os.path.isfile(p) and os.access(p, os.X_OK)), None) + return next((p for p in candidate_paths if os.path.isfile(p) and os.access(p, os.X_OK)), None) diff --git a/problemtools/run/viva.py b/problemtools/run/viva.py index b6129169..23e35c68 100644 --- a/problemtools/run/viva.py +++ b/problemtools/run/viva.py @@ -9,8 +9,8 @@ class Viva(Executable): - """Wrapper class for running VIVA scripts. - """ + """Wrapper class for running VIVA scripts.""" + _VIVA_PATH = get_tool_path('viva.sh') def __init__(self, path): @@ -20,32 +20,25 @@ def __init__(self, path): path (str): path to .viva source file """ if Viva._VIVA_PATH is None: - raise ProgramError( - 'Could not locate the VIVA program to run %s' % path) - super(Viva, self).__init__(Viva._VIVA_PATH, - args=[path]) - + raise ProgramError('Could not locate the VIVA program to run %s' % path) + super().__init__(Viva._VIVA_PATH, args=[path]) def __str__(self): """String representation""" return '%s' % (self.args[0]) - - _compile_result = None - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Syntax-check the VIVA script Returns: (False, None) if the VIVA script has syntax errors and (True, None) otherwise """ - if self._compile_result is None: - (status, _) = super(Viva, self).run() - self._compile_result = ((os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0), None) - return self._compile_result - + (status, _) = super().run() + return ((os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0), None) - def run(self, infile='/dev/null', outfile='/dev/null', - errfile='/dev/null', args=None, timelim=1000): + def run( + self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', args=None, timelim=1000, memlim=1024, work_dir=None + ): """Run the VIVA script to validate an input file. Args: @@ -69,14 +62,13 @@ def run(self, infile='/dev/null', outfile='/dev/null', if infile != '/dev/null': args = args + [infile] - (status, runtime) = super(Viva, self).run(outfile=outfile, - errfile=errfile, - args=args, - timelim=timelim) + (status, runtime) = super(Viva, self).run( + outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir + ) # This is ugly, switches the accept exit status and our accept # exit status 42. if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: - return (42<<8, runtime) + return (42 << 8, runtime) if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: return (0, runtime) return (status, runtime) diff --git a/problemtools/statement_util.py b/problemtools/statement_util.py new file mode 100644 index 00000000..c5a3f6a8 --- /dev/null +++ b/problemtools/statement_util.py @@ -0,0 +1,335 @@ +import collections +import html +import json +import os +import re +import subprocess +import tempfile +import logging +from pathlib import Path +from typing import Optional, List, Tuple +from urllib.parse import urlparse + +from . import metadata +from .formatversion import FormatVersion, get_format_version + +log = logging.getLogger(__name__) + +ALLOWED_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg') # ".svg" +FOOTNOTES_STRINGS = ['
', '